Imports¶
In [108]:
#!/usr/bin/env python
from __future__ import print_function
import platform
import sys
import os
import warnings
warnings.simplefilter(action='ignore')
import logging
#print all logging.INFO details
#logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
#Supress default INFO logging
logger = logging.getLogger()
logger.setLevel(logging.CRITICAL)
from time import time
from itertools import groupby
from more_itertools import unique_everseen
from operator import itemgetter
import bs4
from bs4 import BeautifulSoup
import urllib
from urllib.request import urlopen
import re
import regex
import string
import textwrap
from textwrap import fill
#import pycontractions
#from pycontractions import Contractions
import autocorrect
from autocorrect import spell
import numpy as np
import pandas as pd
import textacy
from textacy.preprocess import remove_punct
import spacy
from spacy.tokenizer import Tokenizer
import gensim
from gensim.matutils import hellinger
from gensim import corpora, models, similarities
from gensim.corpora import Dictionary
from gensim.models import Doc2Vec, CoherenceModel, LdaModel, HdpModel, LsiModel
from gensim.models.wrappers import LdaMallet #,LdaVowpalWabbit
import lda
import pyLDAvis.gensim
pyLDAvis.enable_notebook()
import IPython
from IPython.display import display
import sklearn
from sklearn import metrics
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, TfidfTransformer
from sklearn.preprocessing import normalize
from sklearn.preprocessing import MinMaxScaler
from sklearn.decomposition import PCA, NMF, TruncatedSVD #,LatentDirichletAllocation
from sklearn.manifold import TSNE
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec
%matplotlib inline
import plotly
from plotly import tools
import plotly.plotly as py
import plotly.graph_objs as go
import plotly.figure_factory as ff
from plotly.offline import download_plotlyjs,init_notebook_mode,plot,iplot
#connects JS to notebook so plots work inline
init_notebook_mode(connected=True)
import cufflinks as cf
#allow offline use of cufflinks
cf.go_offline()
import bokeh
from bokeh.io import push_notebook, show, output_notebook
import bokeh.plotting as bp
from bokeh.plotting import figure, save
from bokeh.models import ColumnDataSource, LabelSet, HoverTool
output_notebook()
Environment and Library Information¶
In [3]:
#environment and package versions
print('\n')
print("_"*70)
print('The environment and package versions used in this script are:')
print('\n')
print(platform.platform())
print('Python', sys.version)
print('OS', os.name)
print('Beautiful Soup', bs4.__version__)
print('Urllib', urllib.request.__version__)
print('Regex', re.__version__)
print('Numpy', np.__version__)
print('Pandas', pd.__version__)
print('Textacy', textacy.__version__)
print('SpaCy', spacy.__version__)
print('Gensim', gensim.__version__)
print('LDA', lda.__version__)
print('PyLDAvis', pyLDAvis.__version__)
print('IPython', IPython.__version__)
print('Sklearn', sklearn.__version__)
print('Matplotlib', matplotlib.__version__)
print('Plotly', plotly.__version__)
print('Cufflinks', cf.__version__)
print('Bokeh', bokeh.__version__)
print('\n')
print("~"*70)
print('\n')
______________________________________________________________________ The environment and package versions used in this script are: Windows-10-10.0.17134-SP0 Python 3.6.6 |Anaconda custom (64-bit)| (default, Jun 28 2018, 11:27:44) [MSC v.1900 64 bit (AMD64)] OS nt Beautiful Soup 4.6.0 Urllib 3.6 Regex 2.2.1 Numpy 1.15.4 Pandas 0.22.0 Textacy 0.6.2 SpaCy 2.0.12 Gensim 3.6.0 LDA 1.1.0 PyLDAvis 2.1.2 IPython 5.1.0 Sklearn 0.19.1 Matplotlib 2.2.2 Plotly 2.2.2 Cufflinks 0.12.1 Bokeh 0.12.4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Helper Functions¶
In [4]:
def get_movie_reviews(soup_broth, n_reviews_per_movie=1):
print('Retrieving all baseline URLs... \n')
base_urls = [ ("https://www.imdb.com" + tag.attrs['href'], tag.text.replace('\n',' ') )
for tag in soup_broth.findAll('a', attrs={'href': re.compile("^/title/.*_tt")}) ]
print('Retrieved all second-level URLs... \n')
level_2_urls = []
for url, title in base_urls:
soup = BeautifulSoup(urlopen(url).read(), 'html.parser')
update_url = [("https://www.imdb.com" + tag.attrs['href'])
for tag in soup.findAll('a', attrs={'href': re.compile("^/title/.*tt_urv")})]
level_2_urls.append((update_url[0], title))
print('Retrieved all third-level URLs... \n')
level_3_urls = []
for url, title in level_2_urls:
soup = BeautifulSoup(urlopen(url).read(), 'html.parser')
update_url = [("https://www.imdb.com" + tag.attrs['href'])
for tag in soup.findAll('a', attrs={'href': re.compile("^/review/.*tt_urv")})[:(n_reviews_per_movie*2)]]
update_url = list(unique_everseen(update_url))
for i in update_url:
level_3_urls.append((i, title))
print('Retrieved all fourth-level URLs... \n')
level_4_urls = []
for url, title in level_3_urls:
soup = BeautifulSoup(urlopen(url).read(), 'html.parser')
update_url = [("https://www.imdb.com" + soup.find('a', href=re.compile("^/review/.*rw_urv"))['href'])]
level_4_urls.append((update_url[0], title))
print('Retrieved all fifth-level URLs... \n')
level_5_text = []
for url, title in level_4_urls:
soup = BeautifulSoup(urlopen(url).read(), 'html.parser')
review_text = [(soup.find('div', {'class' : re.compile("^text")}).text)]
level_5_text.append((review_text[0], title))
print('All reviews retrieved! \n')
return level_5_text
In [5]:
def clean_component(review, stop_words, tokenizer, puncts):
"""Text Cleaner: Tokenize, Remove Stopwords, Punctuation, Lemmatize, Spell Correct, Lowercase"""
#rev_contract_exp = list(contract_model.expand_texts([review], precise=True))
doc_tok = tokenizer(review)
doc_lems = [tok.lemma_ for tok in doc_tok
if (tok.text not in stop_words
and tok.text not in puncts
and tok.pos_ != "PUNCT" and tok.pos_ != "SYM")]
lem_list = [re.search(r'\(?([0-9A-Za-z-]+)\)?', tok).group(1)
if '-' in tok
else spell(remove_punct(tok))
for tok in doc_lems]
doc2vec_input = [t.lower() for tok in lem_list
for t in tok.split()
if t.lower() not in stop_words]
return doc2vec_input
In [6]:
def get_tagged_documents(input_review_texts, stop_words, tokenizer, puncts, sentence_labeler):
print('Creating Tagged Documents... \n')
all_content = []
j=0
for rev, ttl in input_review_texts:
print('Cleaning review #{}'.format(j+1))
clean_rev = clean_component(rev, stop_words, tokenizer, puncts)
all_content.append(sentence_labeler(clean_rev, [ttl]))
j += 1
print('Total Number of Movie Review Document Vectors: ', j)
return all_content
In [39]:
def compute_coherence_values(model_type, dictionary, texts, max_topics,
corpus=False, topics=False, transformed_vectorizer=False, tfidf_norm=False,
min_topics=2, stride=3, n_top_words=False, measure='u_mass',
lsi_flag=False, nmf_flag=False, mallet_flag=False, mallet_path=False):
coherence_values = []
model_list = []
if nmf_flag:
feat_names = transformed_vectorizer.get_feature_names()
for num_topics in range(min_topics, max_topics, stride):
if lsi_flag:
model=model_type(corpus=corpus, id2word=dictionary, num_topics=num_topics)
elif nmf_flag:
model=model_type(n_components=num_topics, init='nndsvd', random_state=111)
elif mallet_flag:
model=model_type(mallet_path=mallet_path, corpus=corpus, id2word=dictionary, num_topics=num_topics)
else:
model=model_type(corpus=corpus, id2word=dictionary, num_topics=num_topics, random_state=121)
model_list.append(model)
if nmf_flag:
words_list = []
for i in range(num_topics):
model.fit(tfidf_norm)
#for each topic, obtain the largest values, and add the words they map to into the dictionary
words_ids = model.components_[i].argsort()[:-n_top_words - 1:-1]
words = [feat_names[key] for key in words_ids]
words_list.append(words)
coherencemodel = CoherenceModel(topics=words_list, texts=texts, dictionary=dictionary, coherence=measure)
else:
coherencemodel = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence=measure)
coherence_values.append(coherencemodel.get_coherence())
print('Coherence of model with {} topics has been computed'.format(num_topics))
print('All coherence values have been computed for the {} using the {} measure'.format(model_type, measure.upper()))
print('\n')
return model_list, coherence_values
In [8]:
def show_best_num_topics(model_type, umass_coherence_vals, cv_coherence_vals, max_topics, min_topics=2, stride=3):
min_topics=min_topics
max_topics=max_topics
stride=stride
x = range(min_topics, max_topics, stride)
max_y1 = max(umass_coherence_vals)
max_x1 = x[umass_coherence_vals.index(max_y1)]
max_y2 = max(cv_coherence_vals)
max_x2 = x[cv_coherence_vals.index(max_y2)]
fig = plt.figure(constrained_layout=True, figsize=(14,5))
gs = GridSpec(1, 2, figure=fig)
ax1 = fig.add_subplot(gs[0, 0])
ax2 = fig.add_subplot(gs[0, -1])
ax1.plot(x, umass_coherence_vals, label='Coherence Values')
ax1.set_xlabel('Num Topics')
ax1.set_ylabel('Coherence score (U_MASS)')
ax1.legend(loc='best')
ax1.text(max_x1, max_y1, str((max_x1, max_y1)))
ax2.plot(x, cv_coherence_vals, label='Coherence Values')
ax2.set_xlabel('Num Topics')
ax2.set_ylabel('Coherence score (C_V)')
ax2.legend(loc='best')
ax2.text(max_x2, max_y2, str((max_x2, max_y2)))
fig.suptitle('Coherence Scores ({})'.format(model_type))
plt.show()
print('The most coherent number of {} topics using the U_MASS measure is: {}'.format(model_type, max_x1))
print('The most coherent number of {} topics using the C_V measure is: {}'.format(model_type, max_x2))
print('\n')
tup_list = [(max_x1, max_y1), (max_x2, max_y2)]
best_num_topics = max(tup_list, key=itemgetter(1))[0]
return best_num_topics
In [9]:
def get_topics(model, num_topics):
word_dict = {}
for i in range(num_topics):
words = model.show_topic(i, topn = 20)
word_dict['Topic # ' + '{:02d}'.format(i+1)] = [i[0] for i in words]
return pd.DataFrame(word_dict)
In [10]:
def get_nmf_topics(model, num_topics, n_top_words):
#the word ids obtained need to be reverse-mapped to the words so we can print the topic names.
feat_names = vectorizer.get_feature_names()
word_dict = {}
for i in range(num_topics):
#for each topic, obtain the largest values, and add the words they map to into the dictionary.
words_ids = model.components_[i].argsort()[:-n_top_words - 1:-1]
words = [feat_names[key] for key in words_ids]
word_dict['Topic # ' + '{:02d}'.format(i+1)] = words
return pd.DataFrame(word_dict)
In [11]:
def topics_2_bow(topic, model, lsi_flag=False, mallet_flag=False):
topic = topic.split('+')
topic_bow = []
lsi_mallet_array = np.array([])
lsi_mallet_dict = {}
lsi_mallet_dict_scaled = {}
for word in topic:
#split probability and word
try:
prob, word = word.split('*')
except:
continue
#replace unwanted characters
rep = {' ': '', '"': ''}
replace = dict((re.escape(k), v) for k, v in rep.items())
pattern = re.compile("|".join(replace.keys()))
word = pattern.sub(lambda m: replace[re.escape(m.group(0))], word)
#convert to word_type
try:
word = model.id2word.doc2bow([word])[0][0]
except:
continue
if lsi_flag or mallet_flag:
lsi_mallet_array = np.append(lsi_mallet_array, float(prob))
lsi_mallet_dict.update({word:prob})
else:
topic_bow.append((word, float(prob)))
if lsi_flag or mallet_flag:
scaler = MinMaxScaler(feature_range=(0,1), copy=True)
lsi_mallet_scaled = scaler.fit_transform(lsi_mallet_array.reshape(-1,1))
lsi_mallet_scaled = (lsi_mallet_scaled - lsi_mallet_scaled.min()) / (lsi_mallet_scaled - lsi_mallet_scaled.min()).sum()
for k,v in zip(lsi_mallet_dict.keys(), lsi_mallet_scaled):
lsi_mallet_dict_scaled.update({k:v[0]})
for k,v in lsi_mallet_dict_scaled.items():
topic_bow.append((k,v))
return topic_bow
In [12]:
def get_most_similar_topics(model, topics_df=False, num_topics=False, mallet_flag=False, hdp_flag=False, lsi_flag=False, nmf_flag=False, columns=False):
if not nmf_flag:
mod_topics = tuple(topics_df.columns)
mod_top_dict = {}
if hdp_flag:
for k,v in zip(mod_topics, model.show_topics(num_words=len(model.id2word))):
mod_top_dict.update({k:v})
else:
for k,v in zip(mod_topics, model.show_topics(num_words=len(model.id2word), num_topics=num_topics)):
mod_top_dict.update({k:v})
for k,v in mod_top_dict.items():
mod_top_dict[k] = topics_2_bow(v[1], model, lsi_flag, mallet_flag)
else:
mod_topics = tuple(columns)
mod_top_dict = {}
nmf_top_list = []
for k,v in zip(mod_topics, model.components_):
nmf_top_list.append(tuple((k, v)))
scaler = MinMaxScaler(feature_range=(0,1), copy=True)
for k,v in nmf_top_list:
v_scaled = scaler.fit_transform(v.reshape(-1,1))
v = (v_scaled - v_scaled.min()) / (v_scaled - v_scaled.min()).sum()
mod_top_dict.update({k:v})
hellinger_dists = [(hellinger(mod_top_dict[x], mod_top_dict[y]), x, y)
for i,x in enumerate(mod_top_dict.keys())
for j,y in enumerate(mod_top_dict.keys())
if i != j]
unique_hellinger = [tuple(x) for x in set(map(frozenset, hellinger_dists)) if len(tuple(x)) == 3]
resorted_hellinger = []
for i in range(len(unique_hellinger)):
resorted_hellinger.append(sorted(tuple(str(e) for e in unique_hellinger[i])))
resorted_hellinger[i][0] = float(resorted_hellinger[i][0])
resorted_hellinger[i] = tuple(resorted_hellinger[i])
resorted_hellinger = sorted(resorted_hellinger, reverse=True)
return resorted_hellinger
In [13]:
def get_dominant_topics(model, corpus, texts, lsi_flag=False):
dom_topics_df = pd.DataFrame()
#for all topic/topic-probability pairings
for i, row in enumerate(model[corpus]):
#return the pairings, sorting first the one with the highest topic-probability in the mixture
if lsi_flag:
row = sorted(row, key=lambda x: abs(x[1]), reverse=True)
else:
row = sorted(row, key=lambda x: (x[1]), reverse=True)
#for every topic/topic-probability pairing in the sorted tuple list
for j, (topic_num, topic_prob) in enumerate(row):
#take the pairing with the highest topic-probability
if j == 0:
#return the tuple list of top 'n' word-probabilities associated with that topic
wp = model.show_topic(topic_num, topn=20)
#create a list of those top 'n' words
topic_keywords = ", ".join([word for word, prob in wp])
#append to the empty dataframe a series containing:
# the dominant topic allocation for that document,
# the topic-probability it contributes to that document,
# and the top 'n' words associated with that topic
dom_topics_df = dom_topics_df.append(pd.Series([int(topic_num), round(topic_prob,4), topic_keywords]), ignore_index=True)
else:
#ignore other topics in the mixture
break
#name the columns of the constructed dataframe
dom_topics_df.columns = ['Dominant_Topic', 'Probability_Contribution', 'Topic_Keywords']
#append to the original text as another column in the dataframe
contents = pd.Series(texts)
dom_topics_df = pd.concat([dom_topics_df, contents], axis=1)
dom_topics_final = dom_topics_df.reset_index()
dom_topics_final.columns = ['Document_Number', 'Dominant_Topic', 'Probability_Contribution', 'Topic_Keywords', 'Original_Text']
dom_topics_final.set_index('Document_Number', inplace=True)
return dom_topics_df, dom_topics_final
In [14]:
def get_most_representative_docs(dom_topics_final_df, n_topics=20, lsi_flag=False):
representative_docs = pd.DataFrame()
dom_topics_grpd = dom_topics_final_df.groupby('Dominant_Topic')
#for every dominant topic, find the document to which it contributed the greatest (or largest magnitude [LSI]) probability contribution
if not lsi_flag:
for i, grp in dom_topics_grpd:
representative_docs = pd.concat([representative_docs,
grp.sort_values(['Probability_Contribution'], ascending=True).head(1)],
axis=0)
else:
for i, grp in dom_topics_grpd:
representative_docs = pd.concat([representative_docs,
#sort by descending absolute value
grp.reindex(grp.Probability_Contribution.abs().sort_values(inplace=False, ascending=False).index).head(1)],
axis=0)
representative_docs.reset_index(drop=True, inplace=True)
representative_docs.columns = ['Topic_Number', 'Probability_Contribution', 'Topic_Keywords', 'Most_Representative_Document']
representative_docs['Topic_Number'] = representative_docs['Topic_Number'].astype(int)
representative_docs.set_index('Topic_Number', inplace=True)
rep_docs_first_n = representative_docs.head(n_topics)
return rep_docs_first_n
In [15]:
def get_topic_distribution(dom_topics_final_df, rep_doc_df=False, n_topics=20, hdp_flag=False):
if hdp_flag:
rep_doc_df = get_most_representative_docs(dom_topics_final_df, n_topics=dom_topics_final_df['Dominant_Topic'].nunique())
rep_doc_df.reset_index(inplace=True)
#topic number and keywords
topic_num_keywords = rep_doc_df[['Topic_Number', 'Topic_Keywords']]
#number of documents allocated to each topic
topic_counts = dom_topics_final_df['Dominant_Topic'].value_counts()
#topic allocation percentage of total corpus
topic_percent = round(topic_counts/topic_counts.sum(), 4)
#concat number of allocated docs and percentage
topic_dist = pd.concat([topic_num_keywords, topic_counts, topic_percent], axis=1)
topic_dist.columns = ['Topic_Number', 'Topic_Keywords', 'Documents_per_Topic', 'Percent_of_Total_Corpus']
topic_dist.dropna(axis=0, how='any', inplace=True)
topic_dist['Topic_Number'] = topic_dist['Topic_Number'].astype(int)
topic_dist.set_index('Topic_Number', inplace=True)
topic_dist = topic_dist[['Documents_per_Topic', 'Percent_of_Total_Corpus', 'Topic_Keywords']]
topic_dist_first_n = topic_dist.head(n_topics)
return topic_dist_first_n
In [16]:
def svd_2D_scatter(svd_2D_transformed, color_scale_dimension=0):
trace = go.Scattergl(
x = svd_2D_transformed[:,0],
y = svd_2D_transformed[:,1],
mode = 'markers',
marker = dict(
color = svd_2D_transformed[:,color_scale_dimension],
colorscale='Viridis',
colorbar=dict(title='TF-IDF Variation: SVD Component #{}'.format(color_scale_dimension+1)),
line = dict(width = 1)
),
text = vectorizer.get_feature_names(),
)
data = [trace]
iplot(data, filename='scatter-mode')
return None
In [17]:
def svd_3D_scatter(svd_3D_transformed, color_scale_dimension=0):
svd_3D_dim_range = np.stack((np.amax(svd_3D_transformed, axis=0), np.amin(svd_3D_transformed, axis=0)))
svd_3D_max_dim = max(np.max(svd_3D_dim_range, axis=0))
svd_3D_max_dim += svd_3D_max_dim*(0.1)
fig = tools.make_subplots(rows=1, cols=1,
print_grid=False,
specs=[[{'is_3d': True}]])
scene = dict(
camera = dict(
up=dict(x=0, y=0, z=1),
center=dict(x=0, y=0, z=0),
eye=dict(x=2.5, y=0.1, z=0.1)
),
xaxis=dict(
range=[-svd_3D_max_dim, svd_3D_max_dim],
title='SVD_C1',
gridcolor='rgb(255, 255, 255)',
zerolinecolor='rgb(255, 255, 255)',
showbackground=True,
backgroundcolor='rgb(230, 230,230)',
showticklabels=False, ticks=''
),
yaxis=dict(
range=[-svd_3D_max_dim, svd_3D_max_dim],
title='SVD_C2',
gridcolor='rgb(255, 255, 255)',
zerolinecolor='rgb(255, 255, 255)',
showbackground=True,
backgroundcolor='rgb(230, 230,230)',
showticklabels=False, ticks=''
),
zaxis=dict(
range=[-svd_3D_max_dim, svd_3D_max_dim],
title='SVD_C3',
gridcolor='rgb(255, 255, 255)',
zerolinecolor='rgb(255, 255, 255)',
showbackground=True,
backgroundcolor='rgb(230, 230,230)',
showticklabels=False, ticks=''
)
)
centers = [[1, 1], [-1, -1], [1, -1]]
X = svd_3D_transformed
trace = go.Scatter3d(x=X[:, 0], y=X[:, 1], z=X[:, 2],
showlegend=False,
mode='markers',
marker=dict(
color=X[:,color_scale_dimension],
colorscale='Viridis',
colorbar=dict(title='TF-IDF Variation: SVD Component #{}'.format(color_scale_dimension+1)),
line=dict(color='black', width=1)),
text = vectorizer.get_feature_names())
fig.append_trace(trace, 1, 1)
fig['layout'].update(height=1000, width=1200,
margin=dict(l=10,r=10))
fig['layout']['scene'].update(scene)
iplot(fig)
return None
In [18]:
def plot_tsne(doc_list, fitted_lda, fitted_count_vectorizer, transformed_lda, transformed_tsne, color_map, n_top_words=10):
n_top_words = n_top_words
color_map = color_map
#retrieve component key words
_lda_keys = []
for i in range(transformed_lda.shape[0]):
_lda_keys += transformed_lda[i].argmax(),
topic_summaries = []
#matrix of shape n_topics x len(vocabulary)
topic_words = fitted_lda.topic_word_
#all vocab words (strings)
vocab = fitted_count_vectorizer.get_feature_names()
for i, topic_dist in enumerate(topic_words):
#np.argsort returns indices that would sort an array
#iterates over topic component vectors, sorts array in asc order, appends key words from end of array
topic_word = np.array(vocab)[np.argsort(topic_dist)][:-(n_top_words + 1):-1]
topic_summaries.append(' '.join(topic_word))
num_example = len(transformed_lda)
plot_dict = {
'x': transformed_tsne[:, 0],
'y': transformed_tsne[:, 1],
'colors': color_map[_lda_keys][:num_example],
'content': doc_list[:num_example],
'topic_key': _lda_keys[:num_example]
}
#create dataframe from dictionary
plot_df = pd.DataFrame.from_dict(plot_dict)
source = bp.ColumnDataSource(data=plot_df)
title = 'LDA T-SNE Visualization'
plot_lda = bp.figure(plot_width=1400, plot_height=1100,
title=title,
tools="pan,wheel_zoom,box_zoom,reset,hover,previewsave",
x_axis_type=None, y_axis_type=None, min_border=1)
plot_lda.scatter('x','y', color='colors', source=source)
topic_coord = np.empty((transformed_lda.shape[1], 2)) * np.nan
for topic_num in _lda_keys:
if not np.isnan(topic_coord).any():
break
topic_coord[topic_num] = transformed_tsne[_lda_keys.index(topic_num)]
#plot key words
for i in range(transformed_lda.shape[1]):
plot_lda.text(topic_coord[i, 0], topic_coord[i, 1], [topic_summaries[i]])
#hover tools
hover = plot_lda.select(dict(type=HoverTool))
hover.tooltips = {"content": "@content - topic: @topic_key"}
# save the plot
#save(plot_lda, '{}.html'.format(title))
#Cf. JSON Serialization issue:
# https://github.com/bokeh/bokeh/issues/5439
# https://github.com/bokeh/bokeh/issues/6222
# https://github.com/bokeh/bokeh/issues/7523
try:
show(plot_lda, notebook_handle=True)
except Exception as e:
print('Note!: {}'.format(e.__doc__))
print(e)
return None
In [19]:
#dataframe styling
def color_green(val):
color = 'green' if isinstance(val, float) else 'black'
return 'color: {col}'.format(col=color)
def make_bold(val):
weight = 700 if isinstance(val, float) else 400
return 'font-weight: {wt}'.format(wt=weight)
In [20]:
def pretty_print(input_text):
pieces = [str(word) for word in input_text]
output = ' '.join(pieces)
#write_up = fill(output)
print('Hellinger Distances between topics are: \n')
print('_'*70)
print('\n')
print(output.translate({ord(')'):')\n'}))
print('_'*70)
#print(write_up)
return None
Loading Language, Punctuation, and Tagged Document Models¶
In [21]:
nlp = spacy.load('en')
#cont = Contractions('../GoogleNews-vectors-negative300.bin.gz')
#cont.load_models()
punctuations = string.punctuation
LabeledSentence1 = gensim.models.doc2vec.TaggedDocument
Loading Tokenizer and Updating Stop Words¶
In [22]:
tokenizer = Tokenizer(nlp.vocab)
#stopwords = spacy.lang.en.STOP_WORDS
#spacy.lang.en.STOP_WORDS.add("e.g.")
#nlp.vocab['the'].is_stop
nlp.Defaults.stop_words |= {"(a)", "(b)", "(c)", "etc", "etc.", "etc.)", "w/e", "(e.g.", "no?", "s",
"film", "movie","0","1","2","3","4","5","6","7","8","9","10","e","f","k","n","q",
"de","oh","ones","miike","http","imdb", "horror", "like", "good", "great", "little",
"come", "way", "know", "michael", "lot", "thing", "films", "later", "actually", "find",
"big", "long", "away", "filmthe", "www", "com", "x", "aja", "agritos", "lon", "therebravo",
"gou", "b", "particularly", "probably", "sure", "greenskeeper", "try",
"half", "intothe", "especially", "exactly", "20", "ukr", "thatll", "darn", "certainly", "simply", }
stopwords = list(nlp.Defaults.stop_words)
Retrieving and Cleaning Reviews of the Best Horror Movies¶
In [24]:
#open the base URL webpage
html_page = urlopen("https://www.imdb.com/list/ls059633855/")
#instantiate beautiful soup object of the html page
soup = BeautifulSoup(html_page, 'lxml')
review_text_first_5 = get_movie_reviews(soup, n_reviews_per_movie=5)
Retrieving all baseline URLs... Retrieved all second-level URLs... Retrieved all third-level URLs... Retrieved all fourth-level URLs... Retrieved all fifth-level URLs... All reviews retrieved!
In [27]:
all_good_movie_reviews_500 = get_tagged_documents(review_text_first_5, stopwords, tokenizer, punctuations, LabeledSentence1)
Creating Tagged Documents... Cleaning review #1 Cleaning review #2 Cleaning review #3 Cleaning review #4 Cleaning review #5 Cleaning review #6 Cleaning review #7 Cleaning review #8 Cleaning review #9 Cleaning review #10 Cleaning review #11 Cleaning review #12 Cleaning review #13 Cleaning review #14 Cleaning review #15 Cleaning review #16 Cleaning review #17 Cleaning review #18 Cleaning review #19 Cleaning review #20 Cleaning review #21 Cleaning review #22 Cleaning review #23 Cleaning review #24 Cleaning review #25 Cleaning review #26 Cleaning review #27 Cleaning review #28 Cleaning review #29 Cleaning review #30 Cleaning review #31 Cleaning review #32 Cleaning review #33 Cleaning review #34 Cleaning review #35 Cleaning review #36 Cleaning review #37 Cleaning review #38 Cleaning review #39 Cleaning review #40 Cleaning review #41 Cleaning review #42 Cleaning review #43 Cleaning review #44 Cleaning review #45 Cleaning review #46 Cleaning review #47 Cleaning review #48 Cleaning review #49 Cleaning review #50 Cleaning review #51 Cleaning review #52 Cleaning review #53 Cleaning review #54 Cleaning review #55 Cleaning review #56 Cleaning review #57 Cleaning review #58 Cleaning review #59 Cleaning review #60 Cleaning review #61 Cleaning review #62 Cleaning review #63 Cleaning review #64 Cleaning review #65 Cleaning review #66 Cleaning review #67 Cleaning review #68 Cleaning review #69 Cleaning review #70 Cleaning review #71 Cleaning review #72 Cleaning review #73 Cleaning review #74 Cleaning review #75 Cleaning review #76 Cleaning review #77 Cleaning review #78 Cleaning review #79 Cleaning review #80 Cleaning review #81 Cleaning review #82 Cleaning review #83 Cleaning review #84 Cleaning review #85 Cleaning review #86 Cleaning review #87 Cleaning review #88 Cleaning review #89 Cleaning review #90 Cleaning review #91 Cleaning review #92 Cleaning review #93 Cleaning review #94 Cleaning review #95 Cleaning review #96 Cleaning review #97 Cleaning review #98 Cleaning review #99 Cleaning review #100 Cleaning review #101 Cleaning review #102 Cleaning review #103 Cleaning review #104 Cleaning review #105 Cleaning review #106 Cleaning review #107 Cleaning review #108 Cleaning review #109 Cleaning review #110 Cleaning review #111 Cleaning review #112 Cleaning review #113 Cleaning review #114 Cleaning review #115 Cleaning review #116 Cleaning review #117 Cleaning review #118 Cleaning review #119 Cleaning review #120 Cleaning review #121 Cleaning review #122 Cleaning review #123 Cleaning review #124 Cleaning review #125 Cleaning review #126 Cleaning review #127 Cleaning review #128 Cleaning review #129 Cleaning review #130 Cleaning review #131 Cleaning review #132 Cleaning review #133 Cleaning review #134 Cleaning review #135 Cleaning review #136 Cleaning review #137 Cleaning review #138 Cleaning review #139 Cleaning review #140 Cleaning review #141 Cleaning review #142 Cleaning review #143 Cleaning review #144 Cleaning review #145 Cleaning review #146 Cleaning review #147 Cleaning review #148 Cleaning review #149 Cleaning review #150 Cleaning review #151 Cleaning review #152 Cleaning review #153 Cleaning review #154 Cleaning review #155 Cleaning review #156 Cleaning review #157 Cleaning review #158 Cleaning review #159 Cleaning review #160 Cleaning review #161 Cleaning review #162 Cleaning review #163 Cleaning review #164 Cleaning review #165 Cleaning review #166 Cleaning review #167 Cleaning review #168 Cleaning review #169 Cleaning review #170 Cleaning review #171 Cleaning review #172 Cleaning review #173 Cleaning review #174 Cleaning review #175 Cleaning review #176 Cleaning review #177 Cleaning review #178 Cleaning review #179 Cleaning review #180 Cleaning review #181 Cleaning review #182 Cleaning review #183 Cleaning review #184 Cleaning review #185 Cleaning review #186 Cleaning review #187 Cleaning review #188 Cleaning review #189 Cleaning review #190 Cleaning review #191 Cleaning review #192 Cleaning review #193 Cleaning review #194 Cleaning review #195 Cleaning review #196 Cleaning review #197 Cleaning review #198 Cleaning review #199 Cleaning review #200 Cleaning review #201 Cleaning review #202 Cleaning review #203 Cleaning review #204 Cleaning review #205 Cleaning review #206 Cleaning review #207 Cleaning review #208 Cleaning review #209 Cleaning review #210 Cleaning review #211 Cleaning review #212 Cleaning review #213 Cleaning review #214 Cleaning review #215 Cleaning review #216 Cleaning review #217 Cleaning review #218 Cleaning review #219 Cleaning review #220 Cleaning review #221 Cleaning review #222 Cleaning review #223 Cleaning review #224 Cleaning review #225 Cleaning review #226 Cleaning review #227 Cleaning review #228 Cleaning review #229 Cleaning review #230 Cleaning review #231 Cleaning review #232 Cleaning review #233 Cleaning review #234 Cleaning review #235 Cleaning review #236 Cleaning review #237 Cleaning review #238 Cleaning review #239 Cleaning review #240 Cleaning review #241 Cleaning review #242 Cleaning review #243 Cleaning review #244 Cleaning review #245 Cleaning review #246 Cleaning review #247 Cleaning review #248 Cleaning review #249 Cleaning review #250 Cleaning review #251 Cleaning review #252 Cleaning review #253 Cleaning review #254 Cleaning review #255 Cleaning review #256 Cleaning review #257 Cleaning review #258 Cleaning review #259 Cleaning review #260 Cleaning review #261 Cleaning review #262 Cleaning review #263 Cleaning review #264 Cleaning review #265 Cleaning review #266 Cleaning review #267 Cleaning review #268 Cleaning review #269 Cleaning review #270 Cleaning review #271 Cleaning review #272 Cleaning review #273 Cleaning review #274 Cleaning review #275 Cleaning review #276 Cleaning review #277 Cleaning review #278 Cleaning review #279 Cleaning review #280 Cleaning review #281 Cleaning review #282 Cleaning review #283 Cleaning review #284 Cleaning review #285 Cleaning review #286 Cleaning review #287 Cleaning review #288 Cleaning review #289 Cleaning review #290 Cleaning review #291 Cleaning review #292 Cleaning review #293 Cleaning review #294 Cleaning review #295 Cleaning review #296 Cleaning review #297 Cleaning review #298 Cleaning review #299 Cleaning review #300 Cleaning review #301 Cleaning review #302 Cleaning review #303 Cleaning review #304 Cleaning review #305 Cleaning review #306 Cleaning review #307 Cleaning review #308 Cleaning review #309 Cleaning review #310 Cleaning review #311 Cleaning review #312 Cleaning review #313 Cleaning review #314 Cleaning review #315 Cleaning review #316 Cleaning review #317 Cleaning review #318 Cleaning review #319 Cleaning review #320 Cleaning review #321 Cleaning review #322 Cleaning review #323 Cleaning review #324 Cleaning review #325 Cleaning review #326 Cleaning review #327 Cleaning review #328 Cleaning review #329 Cleaning review #330 Cleaning review #331 Cleaning review #332 Cleaning review #333 Cleaning review #334 Cleaning review #335 Cleaning review #336 Cleaning review #337 Cleaning review #338 Cleaning review #339 Cleaning review #340 Cleaning review #341 Cleaning review #342 Cleaning review #343 Cleaning review #344 Cleaning review #345 Cleaning review #346 Cleaning review #347 Cleaning review #348 Cleaning review #349 Cleaning review #350 Cleaning review #351 Cleaning review #352 Cleaning review #353 Cleaning review #354 Cleaning review #355 Cleaning review #356 Cleaning review #357 Cleaning review #358 Cleaning review #359 Cleaning review #360 Cleaning review #361 Cleaning review #362 Cleaning review #363 Cleaning review #364 Cleaning review #365 Cleaning review #366 Cleaning review #367 Cleaning review #368 Cleaning review #369 Cleaning review #370 Cleaning review #371 Cleaning review #372 Cleaning review #373 Cleaning review #374 Cleaning review #375 Cleaning review #376 Cleaning review #377 Cleaning review #378 Cleaning review #379 Cleaning review #380 Cleaning review #381 Cleaning review #382 Cleaning review #383 Cleaning review #384 Cleaning review #385 Cleaning review #386 Cleaning review #387 Cleaning review #388 Cleaning review #389 Cleaning review #390 Cleaning review #391 Cleaning review #392 Cleaning review #393 Cleaning review #394 Cleaning review #395 Cleaning review #396 Cleaning review #397 Cleaning review #398 Cleaning review #399 Cleaning review #400 Cleaning review #401 Cleaning review #402 Cleaning review #403 Cleaning review #404 Cleaning review #405 Cleaning review #406 Cleaning review #407 Cleaning review #408 Cleaning review #409 Cleaning review #410 Cleaning review #411 Cleaning review #412 Cleaning review #413 Cleaning review #414 Cleaning review #415 Cleaning review #416 Cleaning review #417 Cleaning review #418 Cleaning review #419 Cleaning review #420 Cleaning review #421 Cleaning review #422 Cleaning review #423 Cleaning review #424 Cleaning review #425 Cleaning review #426 Cleaning review #427 Cleaning review #428 Cleaning review #429 Cleaning review #430 Cleaning review #431 Cleaning review #432 Cleaning review #433 Cleaning review #434 Cleaning review #435 Cleaning review #436 Cleaning review #437 Cleaning review #438 Cleaning review #439 Cleaning review #440 Cleaning review #441 Cleaning review #442 Cleaning review #443 Cleaning review #444 Cleaning review #445 Cleaning review #446 Cleaning review #447 Cleaning review #448 Cleaning review #449 Cleaning review #450 Cleaning review #451 Cleaning review #452 Cleaning review #453 Cleaning review #454 Cleaning review #455 Cleaning review #456 Cleaning review #457 Cleaning review #458 Cleaning review #459 Cleaning review #460 Cleaning review #461 Cleaning review #462 Cleaning review #463 Cleaning review #464 Cleaning review #465 Cleaning review #466 Cleaning review #467 Cleaning review #468 Cleaning review #469 Cleaning review #470 Cleaning review #471 Cleaning review #472 Cleaning review #473 Cleaning review #474 Cleaning review #475 Cleaning review #476 Cleaning review #477 Cleaning review #478 Cleaning review #479 Cleaning review #480 Cleaning review #481 Cleaning review #482 Cleaning review #483 Cleaning review #484 Cleaning review #485 Cleaning review #486 Cleaning review #487 Cleaning review #488 Cleaning review #489 Cleaning review #490 Cleaning review #491 Cleaning review #492 Cleaning review #493 Cleaning review #494 Cleaning review #495 Cleaning review #496 Cleaning review #497 Cleaning review #498 Cleaning review #499 Cleaning review #500 Total Number of Movie Review Document Vectors: 500
Retrieving and Cleaning Reviews of the Worst Horror Movies¶
In [28]:
#open the base URL webpage
html_page_bad = urlopen("https://www.imdb.com/list/ls059633855/")
#instantiate beautiful soup object of the html page
soup_bad = BeautifulSoup(html_page_bad, 'lxml')
review_text_first_5_bad = get_movie_reviews(soup_bad, n_reviews_per_movie=5)
Retrieving all baseline URLs... Retrieved all second-level URLs... Retrieved all third-level URLs... Retrieved all fourth-level URLs... Retrieved all fifth-level URLs... All reviews retrieved!
In [29]:
all_bad_movie_reviews_500 = get_tagged_documents(review_text_first_5_bad, stopwords, tokenizer, punctuations, LabeledSentence1)
Creating Tagged Documents... Cleaning review #1 Cleaning review #2 Cleaning review #3 Cleaning review #4 Cleaning review #5 Cleaning review #6 Cleaning review #7 Cleaning review #8 Cleaning review #9 Cleaning review #10 Cleaning review #11 Cleaning review #12 Cleaning review #13 Cleaning review #14 Cleaning review #15 Cleaning review #16 Cleaning review #17 Cleaning review #18 Cleaning review #19 Cleaning review #20 Cleaning review #21 Cleaning review #22 Cleaning review #23 Cleaning review #24 Cleaning review #25 Cleaning review #26 Cleaning review #27 Cleaning review #28 Cleaning review #29 Cleaning review #30 Cleaning review #31 Cleaning review #32 Cleaning review #33 Cleaning review #34 Cleaning review #35 Cleaning review #36 Cleaning review #37 Cleaning review #38 Cleaning review #39 Cleaning review #40 Cleaning review #41 Cleaning review #42 Cleaning review #43 Cleaning review #44 Cleaning review #45 Cleaning review #46 Cleaning review #47 Cleaning review #48 Cleaning review #49 Cleaning review #50 Cleaning review #51 Cleaning review #52 Cleaning review #53 Cleaning review #54 Cleaning review #55 Cleaning review #56 Cleaning review #57 Cleaning review #58 Cleaning review #59 Cleaning review #60 Cleaning review #61 Cleaning review #62 Cleaning review #63 Cleaning review #64 Cleaning review #65 Cleaning review #66 Cleaning review #67 Cleaning review #68 Cleaning review #69 Cleaning review #70 Cleaning review #71 Cleaning review #72 Cleaning review #73 Cleaning review #74 Cleaning review #75 Cleaning review #76 Cleaning review #77 Cleaning review #78 Cleaning review #79 Cleaning review #80 Cleaning review #81 Cleaning review #82 Cleaning review #83 Cleaning review #84 Cleaning review #85 Cleaning review #86 Cleaning review #87 Cleaning review #88 Cleaning review #89 Cleaning review #90 Cleaning review #91 Cleaning review #92 Cleaning review #93 Cleaning review #94 Cleaning review #95 Cleaning review #96 Cleaning review #97 Cleaning review #98 Cleaning review #99 Cleaning review #100 Cleaning review #101 Cleaning review #102 Cleaning review #103 Cleaning review #104 Cleaning review #105 Cleaning review #106 Cleaning review #107 Cleaning review #108 Cleaning review #109 Cleaning review #110 Cleaning review #111 Cleaning review #112 Cleaning review #113 Cleaning review #114 Cleaning review #115 Cleaning review #116 Cleaning review #117 Cleaning review #118 Cleaning review #119 Cleaning review #120 Cleaning review #121 Cleaning review #122 Cleaning review #123 Cleaning review #124 Cleaning review #125 Cleaning review #126 Cleaning review #127 Cleaning review #128 Cleaning review #129 Cleaning review #130 Cleaning review #131 Cleaning review #132 Cleaning review #133 Cleaning review #134 Cleaning review #135 Cleaning review #136 Cleaning review #137 Cleaning review #138 Cleaning review #139 Cleaning review #140 Cleaning review #141 Cleaning review #142 Cleaning review #143 Cleaning review #144 Cleaning review #145 Cleaning review #146 Cleaning review #147 Cleaning review #148 Cleaning review #149 Cleaning review #150 Cleaning review #151 Cleaning review #152 Cleaning review #153 Cleaning review #154 Cleaning review #155 Cleaning review #156 Cleaning review #157 Cleaning review #158 Cleaning review #159 Cleaning review #160 Cleaning review #161 Cleaning review #162 Cleaning review #163 Cleaning review #164 Cleaning review #165 Cleaning review #166 Cleaning review #167 Cleaning review #168 Cleaning review #169 Cleaning review #170 Cleaning review #171 Cleaning review #172 Cleaning review #173 Cleaning review #174 Cleaning review #175 Cleaning review #176 Cleaning review #177 Cleaning review #178 Cleaning review #179 Cleaning review #180 Cleaning review #181 Cleaning review #182 Cleaning review #183 Cleaning review #184 Cleaning review #185 Cleaning review #186 Cleaning review #187 Cleaning review #188 Cleaning review #189 Cleaning review #190 Cleaning review #191 Cleaning review #192 Cleaning review #193 Cleaning review #194 Cleaning review #195 Cleaning review #196 Cleaning review #197 Cleaning review #198 Cleaning review #199 Cleaning review #200 Cleaning review #201 Cleaning review #202 Cleaning review #203 Cleaning review #204 Cleaning review #205 Cleaning review #206 Cleaning review #207 Cleaning review #208 Cleaning review #209 Cleaning review #210 Cleaning review #211 Cleaning review #212 Cleaning review #213 Cleaning review #214 Cleaning review #215 Cleaning review #216 Cleaning review #217 Cleaning review #218 Cleaning review #219 Cleaning review #220 Cleaning review #221 Cleaning review #222 Cleaning review #223 Cleaning review #224 Cleaning review #225 Cleaning review #226 Cleaning review #227 Cleaning review #228 Cleaning review #229 Cleaning review #230 Cleaning review #231 Cleaning review #232 Cleaning review #233 Cleaning review #234 Cleaning review #235 Cleaning review #236 Cleaning review #237 Cleaning review #238 Cleaning review #239 Cleaning review #240 Cleaning review #241 Cleaning review #242 Cleaning review #243 Cleaning review #244 Cleaning review #245 Cleaning review #246 Cleaning review #247 Cleaning review #248 Cleaning review #249 Cleaning review #250 Cleaning review #251 Cleaning review #252 Cleaning review #253 Cleaning review #254 Cleaning review #255 Cleaning review #256 Cleaning review #257 Cleaning review #258 Cleaning review #259 Cleaning review #260 Cleaning review #261 Cleaning review #262 Cleaning review #263 Cleaning review #264 Cleaning review #265 Cleaning review #266 Cleaning review #267 Cleaning review #268 Cleaning review #269 Cleaning review #270 Cleaning review #271 Cleaning review #272 Cleaning review #273 Cleaning review #274 Cleaning review #275 Cleaning review #276 Cleaning review #277 Cleaning review #278 Cleaning review #279 Cleaning review #280 Cleaning review #281 Cleaning review #282 Cleaning review #283 Cleaning review #284 Cleaning review #285 Cleaning review #286 Cleaning review #287 Cleaning review #288 Cleaning review #289 Cleaning review #290 Cleaning review #291 Cleaning review #292 Cleaning review #293 Cleaning review #294 Cleaning review #295 Cleaning review #296 Cleaning review #297 Cleaning review #298 Cleaning review #299 Cleaning review #300 Cleaning review #301 Cleaning review #302 Cleaning review #303 Cleaning review #304 Cleaning review #305 Cleaning review #306 Cleaning review #307 Cleaning review #308 Cleaning review #309 Cleaning review #310 Cleaning review #311 Cleaning review #312 Cleaning review #313 Cleaning review #314 Cleaning review #315 Cleaning review #316 Cleaning review #317 Cleaning review #318 Cleaning review #319 Cleaning review #320 Cleaning review #321 Cleaning review #322 Cleaning review #323 Cleaning review #324 Cleaning review #325 Cleaning review #326 Cleaning review #327 Cleaning review #328 Cleaning review #329 Cleaning review #330 Cleaning review #331 Cleaning review #332 Cleaning review #333 Cleaning review #334 Cleaning review #335 Cleaning review #336 Cleaning review #337 Cleaning review #338 Cleaning review #339 Cleaning review #340 Cleaning review #341 Cleaning review #342 Cleaning review #343 Cleaning review #344 Cleaning review #345 Cleaning review #346 Cleaning review #347 Cleaning review #348 Cleaning review #349 Cleaning review #350 Cleaning review #351 Cleaning review #352 Cleaning review #353 Cleaning review #354 Cleaning review #355 Cleaning review #356 Cleaning review #357 Cleaning review #358 Cleaning review #359 Cleaning review #360 Cleaning review #361 Cleaning review #362 Cleaning review #363 Cleaning review #364 Cleaning review #365 Cleaning review #366 Cleaning review #367 Cleaning review #368 Cleaning review #369 Cleaning review #370 Cleaning review #371 Cleaning review #372 Cleaning review #373 Cleaning review #374 Cleaning review #375 Cleaning review #376 Cleaning review #377 Cleaning review #378 Cleaning review #379 Cleaning review #380 Cleaning review #381 Cleaning review #382 Cleaning review #383 Cleaning review #384 Cleaning review #385 Cleaning review #386 Cleaning review #387 Cleaning review #388 Cleaning review #389 Cleaning review #390 Cleaning review #391 Cleaning review #392 Cleaning review #393 Cleaning review #394 Cleaning review #395 Cleaning review #396 Cleaning review #397 Cleaning review #398 Cleaning review #399 Cleaning review #400 Cleaning review #401 Cleaning review #402 Cleaning review #403 Cleaning review #404 Cleaning review #405 Cleaning review #406 Cleaning review #407 Cleaning review #408 Cleaning review #409 Cleaning review #410 Cleaning review #411 Cleaning review #412 Cleaning review #413 Cleaning review #414 Cleaning review #415 Cleaning review #416 Cleaning review #417 Cleaning review #418 Cleaning review #419 Cleaning review #420 Cleaning review #421 Cleaning review #422 Cleaning review #423 Cleaning review #424 Cleaning review #425 Cleaning review #426 Cleaning review #427 Cleaning review #428 Cleaning review #429 Cleaning review #430 Cleaning review #431 Cleaning review #432 Cleaning review #433 Cleaning review #434 Cleaning review #435 Cleaning review #436 Cleaning review #437 Cleaning review #438 Cleaning review #439 Cleaning review #440 Cleaning review #441 Cleaning review #442 Cleaning review #443 Cleaning review #444 Cleaning review #445 Cleaning review #446 Cleaning review #447 Cleaning review #448 Cleaning review #449 Cleaning review #450 Cleaning review #451 Cleaning review #452 Cleaning review #453 Cleaning review #454 Cleaning review #455 Cleaning review #456 Cleaning review #457 Cleaning review #458 Cleaning review #459 Cleaning review #460 Cleaning review #461 Cleaning review #462 Cleaning review #463 Cleaning review #464 Cleaning review #465 Cleaning review #466 Cleaning review #467 Cleaning review #468 Cleaning review #469 Cleaning review #470 Cleaning review #471 Cleaning review #472 Cleaning review #473 Cleaning review #474 Cleaning review #475 Cleaning review #476 Cleaning review #477 Cleaning review #478 Cleaning review #479 Cleaning review #480 Cleaning review #481 Cleaning review #482 Cleaning review #483 Cleaning review #484 Cleaning review #485 Cleaning review #486 Cleaning review #487 Cleaning review #488 Cleaning review #489 Cleaning review #490 Cleaning review #491 Cleaning review #492 Cleaning review #493 Cleaning review #494 Cleaning review #495 Cleaning review #496 Cleaning review #497 Cleaning review #498 Cleaning review #499 Cleaning review #500 Total Number of Movie Review Document Vectors: 500
Creating Corpus of Combined Reviews of Best and Worst Horror Movies¶
In [30]:
all_reviews = all_good_movie_reviews_500 + all_bad_movie_reviews_500
In [31]:
all_reviews_text = []
all_reviews_ttl = []
for i in range(len(all_reviews)):
all_reviews_text.append(all_reviews[i][0])
all_reviews_ttl.append(all_reviews[i][1][0])
In [32]:
reg_ex = regex.compile('[^a-zA-Z]')
In [33]:
dictionary = corpora.Dictionary(all_reviews_text)
In [34]:
#remove extremes (similar to the min/max df step used when creating the tf-idf matrix)
#filtered_dict = dictionary.filter_extremes(no_below=1, no_above=0.8)
#convert the dictionary to a bag of words corpus for reference
corpus = [dictionary.doc2bow(text) for text in all_reviews_text]
In [35]:
all_reviews_joined = [' '.join(w) for w in all_reviews_text]
LDA (Latent Dirichlet Allocation)¶
Cf. http://www.jmlr.org/papers/volume3/blei03a/blei03a.pdf¶
Cf. https://nlp.stanford.edu/events/illvi2014/papers/sievert-illvi2014.pdf (LDAvis)¶
Cf. http://svn.aksw.org/papers/2015/WSDM_Topic_Evaluation/public.pdf (Topic Coherence Measures)¶
In [41]:
mod_list_lda_umass, coh_list_lda_umass = compute_coherence_values(LdaModel, dictionary, all_reviews_text, max_topics=20,
corpus=corpus, topics=False, transformed_vectorizer=False, tfidf_norm=False,
min_topics=2, stride=3, n_top_words=False, measure='u_mass')
mod_list_lda_cv, coh_list_lda_cv = compute_coherence_values(LdaModel, dictionary, all_reviews_text, max_topics=20,
corpus=corpus, topics=False, transformed_vectorizer=False, tfidf_norm=False,
min_topics=2, stride=3, n_top_words=False, measure='c_v')
Coherence of model with 2 topics has been computed Coherence of model with 5 topics has been computed Coherence of model with 8 topics has been computed Coherence of model with 11 topics has been computed Coherence of model with 14 topics has been computed Coherence of model with 17 topics has been computed All coherence values have been computed for the <class 'gensim.models.ldamodel.LdaModel'> using the U_MASS measure Coherence of model with 2 topics has been computed Coherence of model with 5 topics has been computed Coherence of model with 8 topics has been computed Coherence of model with 11 topics has been computed Coherence of model with 14 topics has been computed Coherence of model with 17 topics has been computed All coherence values have been computed for the <class 'gensim.models.ldamodel.LdaModel'> using the C_V measure
In [42]:
best_num_lda_topics = show_best_num_topics('LDA', coh_list_lda_umass, coh_list_lda_cv, max_topics=20, min_topics=2, stride=3)
print('The most coherent number of LDA topics to use is: {}'.format(best_num_lda_topics))
The most coherent number of LDA topics using the U_MASS measure is: 2 The most coherent number of LDA topics using the C_V measure is: 8 The most coherent number of LDA topics to use is: 8
In [43]:
lda_mod = models.LdaModel(corpus, num_topics=best_num_lda_topics,
id2word=dictionary,
update_every=0, #batch training
chunksize=1000,
passes=500,
random_state=121)
In [44]:
#Perplexity measure (lower is better)
print('The Perplexity measure of the final LDA model is: {}'.format(lda_mod.log_perplexity(corpus)))
The Perplexity measure of the final LDA model is: -8.298304571058113
In [39]:
#topics_matrix = lda_mod.show_topics(formatted=True, num_words=20)
#topics_matrix = np.array(topics_matrix)
#topics_matrix.shape
In [40]:
#topic_words = topics_matrix[:,1]
#for i in topic_words:
# print([reg_ex.sub('', w) for w in i.split() if len(reg_ex.sub('', w)) > 0])
# print('\n')
In [45]:
lda_df = get_topics(lda_mod, best_num_lda_topics)
#print(lda_df)
lda_df
Out[45]:
| Topic # 01 | Topic # 02 | Topic # 03 | Topic # 04 | Topic # 05 | Topic # 06 | Topic # 07 | Topic # 08 | |
|---|---|---|---|---|---|---|---|---|
| 0 | watch | story | story | character | story | story | story | story |
| 1 | story | time | dont | story | watch | end | watch | character |
| 2 | think | scene | character | people | time | character | scene | time |
| 3 | time | think | time | time | think | time | time | feel |
| 4 | feel | character | act | work | original | plot | love | dont |
| 5 | character | end | work | dont | feel | start | people | scene |
| 6 | scare | feel | people | scene | people | scene | original | people |
| 7 | people | room | feel | love | scene | watch | ghost | think |
| 8 | family | leave | audience | place | play | fan | dont | look |
| 9 | leave | plot | watch | play | end | director | feel | watch |
| 10 | end | people | look | set | plot | ride | doesnt | doesnt |
| 11 | look | look | home | real | work | turn | think | work |
| 12 | ive | watch | scene | soon | love | dead | creepy | scare |
| 13 | want | vampire | genre | life | look | think | ive | director |
| 14 | work | play | scary | feel | director | genre | character | dark |
| 15 | sarah | sense | end | plot | dont | people | remake | right |
| 16 | act | ive | love | violence | new | work | end | play |
| 17 | original | 1408 | director | dark | girl | pretty | zombie | scary |
| 18 | juno | want | scare | want | want | enjoy | want | house |
| 19 | build | kind | think | start | character | look | act | end |
In [46]:
lda_hellinger_topic_distances = get_most_similar_topics(lda_mod, lda_df, num_topics=best_num_lda_topics)
pretty_print(lda_hellinger_topic_distances)
Hellinger Distances between topics are: ______________________________________________________________________ (0.4837110411248617, 'Topic # 01', 'Topic # 06') (0.48174251539487667, 'Topic # 03', 'Topic # 06') (0.4748510044883907, 'Topic # 06', 'Topic # 07') (0.4717990853726296, 'Topic # 04', 'Topic # 06') (0.46271629711555196, 'Topic # 03', 'Topic # 07') (0.46173491768957614, 'Topic # 02', 'Topic # 03') (0.46127626256335685, 'Topic # 02', 'Topic # 06') (0.46001700327696304, 'Topic # 03', 'Topic # 04') (0.4575400998567088, 'Topic # 02', 'Topic # 07') (0.4549526255283364, 'Topic # 01', 'Topic # 04') (0.45447676874309, 'Topic # 02', 'Topic # 04') (0.4544377282672066, 'Topic # 01', 'Topic # 02') (0.4515640911141052, 'Topic # 06', 'Topic # 08') (0.45126237662510776, 'Topic # 04', 'Topic # 07') (0.44308567455525505, 'Topic # 04', 'Topic # 05') (0.4390830521079113, 'Topic # 01', 'Topic # 05') (0.4389738920982188, 'Topic # 01', 'Topic # 07') (0.4387911167272729, 'Topic # 04', 'Topic # 08') (0.43786335685161076, 'Topic # 05', 'Topic # 06') (0.4355284852956487, 'Topic # 02', 'Topic # 05') (0.43441019971552597, 'Topic # 03', 'Topic # 05') (0.4333198334330377, 'Topic # 05', 'Topic # 07') (0.42670160586444283, 'Topic # 01', 'Topic # 03') (0.4234971666361381, 'Topic # 01', 'Topic # 08') (0.4183075835478236, 'Topic # 07', 'Topic # 08') (0.4160318363128511, 'Topic # 03', 'Topic # 08') (0.40846795466068725, 'Topic # 02', 'Topic # 08') (0.39419233251966707, 'Topic # 05', 'Topic # 08') ______________________________________________________________________
In [47]:
lda_mod_dom_topics_df, lda_mod_dom_topics_final = get_dominant_topics(lda_mod, corpus, all_reviews_joined)
In [48]:
#lda_mod_dom_topics_final.style.applymap(color_green).applymap(make_bold)
lda_mod_dom_topics_final_first30 = lda_mod_dom_topics_final.head(30)
lda_mod_dom_topics_final_first30.style.applymap(color_green).applymap(make_bold)
Out[48]:
| Dominant_Topic | Probability_Contribution | Topic_Keywords | Original_Text | |
|---|---|---|---|---|
| Document_Number | ||||
| 0 | 4 | 0.9921 | story, watch, time, think, original, feel, people, scene, play, end, plot, work, love, look, director, dont, new, girl, want, character | coherent plot tell non-straightforward fashion open interpretations leave require audience fill-in open end possibly filling-in require viewings considerable think sense watch wailing time impression kind watch ponder read watch attempt people interpret explain different conclusion coherent underlie plot matt shift piece puzzle attempt recreate coherent narrative piece fit incompetence underlie story possibly intentionally possibly fundamentally inconsistent optical illusion escher drawing appear describe physical object fact dont physical sense accordingly enjoy boil content technically well-made cinematography acting costumes plot sense tell deceptive lure think plot sense matt sufficient thought leave lingering irritate feel dissatisfaction confusion maybe precisely point tell story purpose instill audience feel confusion face sequence event sense life times |
| 1 | 2 | 0.9883 | story, dont, character, time, act, work, people, feel, audience, watch, look, home, scene, genre, scary, end, love, director, scare, think | spoiler review extraordinary hour technically-perfect humanistic fine writer directors business hauteur saw devil cipher close analog suggest david lynches 2001 mulholland drive technically perfect humanistic suspense opus captivate length walk theatre shake head wonder saw film-makers understand secret story tell present story viewer feel share experience protagonist tell story viewer going provide explanation wailing means re-quote writer director numerous interview enigmatically begin wonder nature god -- recommended levels entertainment puzzled clinic engage wont let |
| 2 | 6 | 0.9972 | story, watch, scene, time, love, people, original, ghost, dont, feel, doesnt, think, creepy, ive, character, remake, end, zombie, want, act | wailing open quote bible easy forget fact watch certain point clear purpose quote served warning religious person scare wailing work religious parables overcome lengthy run time tonal imbalanced deliver funny truly terrify mysterious strange small village south korean village plague sickness police think wild mushroom blame police officer jong-goo think stranger meet woman information mysterious man slowly begin fall rabbit hole consume life daughter start sign sickness personal figure trust avoid director hong-jin nap struggle tonal balance time pitch perfect time place tone struggle stay consistent primarily funny brief burst depravity violence comedy work laugh numb times horrific imagery sudden effective balance lose act comedy slapstick fit rest character over-the-top act exaggerate different introduced middle section moment unintentional humor example man strike lighten caller id shaman phone shaman fortunately balance act smartly switch act intense terrify moment comedy offset terrify act start unfold audience gain new appreciation rest start reinterpret certain scenes problem scene sense ended feel natural fit story progressed require end sense hour minutes middle portion resolved shaman screen time perform ritual end important zombie scene feel awkward didnt fit rest want zombie grow popularity scene bring numb random character serve purpose rest despite scene previously mention overacting act generally speak fantastic father task solve happen daughter convey terror hatred build intense persona carry priest train advice father equally effective bring concern innocent quality terror ultimately happen young daughter sick shines channeling inner linda blair emphatically deliver horrible dirty line child performance truly terrify watch hatred eye slowly end religious overtone clearer locusts attack individual white black suggest characters true nature scene truly shines slowly unveil real nature certain characters minute change perspective things minute switch determine trust reveal realize detail setup reveal twist lead ultimately putt faith wrong people wailing 2016 directed hong-jin na screenplay hong-jin na starring woo-hee jeong-min hwang so-yeon range won krak run time hour minute |
| 3 | 7 | 0.988 | story, character, time, feel, dont, scene, people, think, look, watch, doesnt, work, scare, director, dark, right, play, scary, house, end | hell ride minute stop look time leave relieved left thats engage interest story watch read review smorgasbord genre positive review doesnt feel means yes storytelling fluid write multiple genre aspect didnt feel place works work feel redundant point genres similar witch necessarily term storyline pace severity terror psychological wailing deliver somewhat complex storyline cinematography breathtaking jun kunimura ridiculous oh-so-good actor point ism point mention asian technically guess lol dont think necessary pigeonhole |
| 4 | 7 | 0.6144 | story, character, time, feel, dont, scene, people, think, look, watch, doesnt, work, scare, director, dark, right, play, scary, house, end | want start review completely jump scares play integral movies rely support story leave displeased wailing special jump scare instead viewers story rich mystery character struggles dark atmosphere design amazingly craft scene blood run cold addition carry subtext leave viewer question evidence interpret feel director time research real experience possible wailing bite bore slow burner construct modern running time th 36min patient man pay end genre dead look cash grab mainstream wide release hide gems |
| 5 | 3 | 0.9961 | character, story, people, time, work, dont, scene, love, place, play, set, real, soon, life, feel, plot, violence, dark, want, start | ism biased set city live work oxford street piccadilly circus pass morning usually teem crowd people completely send shiver spine usually watch locate nondescript midwestern village easy detach event unfold screen occur place home entirely new sense reality previously unaccustomed judge days entirely merit easy arrive conclusion fantastic achievements coming-of-age sort director danny boyle canst mtv-inspired vanity beach self-consciously trendy posture trainspotting appeal shame initially expect days similar treatment thankfully fear prove unfounded discard straight open sequence effortless fearsome rest joy terrify joy joy nonetheless true minimalism effective overblown bravado definitely true scene complete silence entire metropolis grainy picture serve add documentary-style quality situation real bear definitely wise choice digital video occasionally meet people think days wasnt scary gory people tell 2001 boring memento confusing ignore didnt understand purpose confuse change pace feel distract personally regard important reviewer point succinct point scarier end world world repopulate maniacs think real days lies 28 days romero zombie flick yore ultimately allegory day live age constantly confront violence medium ape right start violence beget violence humanity face uncertain future applaud danny boilers bravery days undoubtedly commercial risk majority cinema-going public prefer escapism word caution remember rage human-made disease allegory masterpiece time days misunderstand considerable people doubt history classic perfectly sum confuse era live didnt advisable days chance haunt experience look right angle danny boyle year leave hope hell continue |
| 6 | 3 | 0.9922 | character, story, people, time, work, dont, scene, love, place, play, set, real, soon, life, feel, plot, violence, dark, want, start | virus rage virus infect person mad extreme rage hungry blood day outbreak london cause entire britain dead evacuate leave blood-thirsty infect population handful solitary normal persons civilization halt society destroy limit survivor fight existence frequent attack vicious victims sounds familiar 28 days classic horde zombie biohazard movies touch art danny boyle able bring focus extensive special-effects-controlled gory action sequence infect normals heavy background music theres tinge sadness emptiness helplessness consider london scene background music theres electrify action gore describe picture life condition talk mistake loophole couldnt weaken tight-gripping storyline achievement viewer stay neutral place virus talk metaphor rage social disease flick person harsh critic zombie watch stars mention billian murphy awesome |
| 7 | 1 | 0.9856 | story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind | ism amaze negative review think succeed level artistic atmospheric pace sympathetic characters fantastic climax music nicely eerie open scene london street worth price admission ism stubborn viewer normally benefit early critical buzz manner excuse time ism completely impressed incidentally think interest day inspire knockoffs knockoffs 28 days apparently owe john wyndham classic disaster novel day triffids thats things highly recommended |
| 8 | 4 | 0.993 | story, watch, time, think, original, feel, people, scene, play, end, plot, work, love, look, director, dont, new, girl, want, character | 2003 state-side release danny boilers 28 days advertise chockful scare-fest didnt day ago gotta feel somewhat embellishment promoters environmental terrorist attack lab contain diseased chimp infect rage virus unwittingly let loose plague lay waste england rest society days title cut abandon london coma-tized bicycle courier jim cillian murphy effective wake stasis hospital search london sign life rescue rage zombie couple survivalists lovely naomi harris follow place place alive meet man daughter brendan gleeson terrific megan burns refuge london-town record message paradise salvation track frank man shortwave radio feel meditation happen people reduce low elements friend tell movies run zombie inspire zombie remake dawn dead dawn pretty full-throttle action hybrid start finish play emphasis creature lucky survivors disturb lesson nature survival interest disturb flick sell wrong |
| 9 | 7 | 0.9971 | story, character, time, feel, dont, scene, people, think, look, watch, doesnt, work, scare, director, dark, right, play, scary, house, end | key sci-fi genre alive cinemas material technique filmmakers present competent average creative havenet havenet style form george romero prime era film-making bring forth memorable trilogy time genre consider romero list director included important source independent 1968 inventive result masterpiece lacklusters director danny boyle author alex garland cook yarn similar romero wouldnt satisfying theyve essential success days later- idea practice years turn fresh audience feel repelled excited terrified nauseous enthralled wont leave feel weave complete hack work boyle team set freshen zombie string precise term zombie movie- hear living-dead utter hear infected new word people rage infect britain monkey virus let loose island begin infectious spread cut man jim lie hospital bed wander abandon street view tear fragment society boyle implement atmosphere heavily action chase scene indication dedication details jim soon survivors include selena naomie harris father daughter brendan gleeson megan burns hear salvation radio decide brave military outpost thats true salvation outside military typos boyle producer andrew macdonald spellbind thats proper terminology adaptation trainspotting craft par arguably toppings story characters captivate care plight jim companions photography anthony dod mantle striking digital photography blair witch use non-professional camera equipment add proper shade needed shoot composition different edit chris gill quick expect attack scenes fast infect throw blood scream on-looker new infect transform seconds result follow mark tildesley production designs john murphys music evoke haunting evocative mood mundane scenes acting consider actor well-known believable script ism days cup tea science fiction fan immediately hear preview tv add dismiss phooey rubble borrow video-store point view similarity romero military scene remind day dead chain zombie practical reasons supermarket scene unneeded consider satirical reverence dawn dead understand boyle isnt 100 original point genres history credible job different tone different set country different envelop view scene structures overall days construct execute sci-fi youve seen sci-fi youve combined sense modern audience fascinate |
| 10 | 7 | 0.6891 | story, character, time, feel, dont, scene, people, think, look, watch, doesnt, work, scare, director, dark, right, play, scary, house, end | let right vampire seen smarter scary nuanced doesnt feel thriller feel literature detail bizarre misadventure pair pre-teen star cross lovers androgynous vampired phenomenal regard detail young oskar kare hedebrant life spot stuck incredibly painful period post-childhood pre-adolescence oskar aware girls idea contend small age brutalize boy result terribly collect news clip violent crime let rage day strange young girl eli klina leandersson appear playground fast friend begin look oskar eli innocently spend night occasion privy oskar doesnt happens elias caretaker serial killer brutal order desanguinating victim bucket soon oskar realize new friend bite tragedy shock violence eli leave fend desperately stave urge drink fresh blood form delicate new bond oskar remake let right dont wait hope capture magic original gorgeously shot impeccably written fully realize unmatched respect vampire lore acting producer amaze young actors odd recapture brilliant melancholy chemistry astronomical let right think traditional opt endless ultra violence cut favor kiddie friendly rating director tomas alfredson steer line right middle violence brutal horrific dwell leave question kidney display stillness mimic oskar stage stuck moving hope growth begin changes apparent stillness oskar elik oskar grow change man eli stick burdensome fate quiet understate ending haunting whimsical forth times people understanding kind people beg dont miss time silence lambs real chance home oscar gold deserve a+ |
| 11 | 1 | 0.9949 | story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind | let right heart sweet coming-of-age story unique different defy categorizations swedish adapt john aside lindqvist bestselling book director tomas alfredson dare mix pleasure pain horrify tender let right storyline reveal secret early leave surprise necessarily rare review plot better 12-year-olds oskar kare hedebrant eli klina leandersson meet snowy afternoon jungle gym courtyard oskar house complex outside stockholm young tender attraction apparent right start think relationship headed deep dark secret discover reveal audience repulse curiously fascinate time similar fashion yellow crime scene tape bring close warn support cast completely beholden narrative revolve adorable young couple performance rival ive actor age innocence vulnerability hedebrant oskar tour-de-force admirably carry shoulders leandersson match scene scene line line result literally chills production value stellary technical aspect -- lighting original music johan soderqvist hoyle van hoytema cinematography -- combine perfect synchronization produce hitchcockian tale bring love light dark drama imaginable let right overwhelm choice best narrative feature north american premiere 2008 tribes festival truly well-deserved honor tomas alfredson craft brilliant work art leave shake head wonder |
| 12 | 3 | 0.9541 | character, story, people, time, work, dont, scene, love, place, play, set, real, soon, life, feel, plot, violence, dark, want, start | tomas alfredson let right original dark twist gory fantasy special hard classify merely exercise style brilliant piece amoral storytellings characters action defy logic common sense dont wanna spoil moment ll mean revenge moment story happens remind fantasy tale oskar kåre hedebrant year-old bully boy befriend develop innocent crush new neighbor eli klina leandersson happen vampired twist tale revenge pubescent love visual flair swim pool scene classic creative direct impressive performance young pair protagonists hollywood course didnt waste time announce upcoming remake lazy read subtitles likely remake turn pg-13 order money fill moral value prudish parent let kid watch dont vampired year ago ignore future bomb enjoy original youre treaty |
| 13 | 1 | 0.9956 | story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind | people review idb focus sweet friendship year old human vampire leads huge element sweet story childhood friendship godfather story father concern worry kids true descriptions miss point think seen truth let right dark measure brilliant story tell week ago disturb echo stay refuse enjoy prior knowledge impossible review properly disclose plot elements warn spoilers ahead comment said character hokan key story end lead believe oskar step place fact reveal final scene dark nightmarish core blossom friendship witness prior offer redemption oskar path relationship eli takes håkan butcher boy old oskar food elik story storytelling high order dark whisper hint suggest questions allude play mind answered hokan pick victim presumably elias preference elias interest oskar elias true nature time briefly glimpse true physical age fact late middle age roughly age håkan gift oskar moments brutal slaughter child tormentors entranced glamour oskar happy happy happily sunset oskar presumably deaden soul destroy husk person hokan butcher fellow human provide eli food puzzle people think sweet fact conclusion utterly chilling oskar happily swap commonplace misery childhood bully fate truly hell earth happily smile shoot doesnt clue superlative heartily recommends tale succeed suggestion play imagination truly addition vampire cannon variation bram stokers brilliant original |
| 14 | 1 | 0.9786 | story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind | fond vampire genre artistic poetic profound explore nature evil world child pure evil pure goodness somewhat discernible achieve astound array contrast allow evil coexist thought-provoking leave audience yearn read novel short gems need remake accord dont need wait 2010 watch |
| 15 | 6 | 0.965 | story, watch, scene, time, love, people, original, ghost, dont, feel, doesnt, think, creepy, ive, character, remake, end, zombie, want, act | zombies zombies cheap outta original idea moves mandatory width train play beautiful girl sucker long-haired woman short skirts babe morality story ethic means played |
| 16 | 6 | 0.9663 | story, watch, scene, time, love, people, original, ghost, dont, feel, doesnt, think, creepy, ive, character, remake, end, zombie, want, act | different zombies army undead create industrial mishap result overwhelm chaos danger address morality root cause personally love pretty woman enjoy leg beautiful hair girl hat |
| 17 | 4 | 0.9687 | story, watch, time, think, original, feel, people, scene, play, end, plot, work, love, look, director, dont, new, girl, want, character | train susan treat mind eyes look capitalism vs humanity man vs empathy remember time enjoy foreign act excellent effeminate young man story original definitely eye delicious cheerleader |
| 18 | 1 | 0.9757 | story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind | dont youths fun action-packed dangerous elements innovative pretty girl importantly successful yup hollywood remake entry series redundant unnecessary inferior whitewash woman western taliban style ugly hollywood remakes paul fig jar jar abrams negotiate cut speak |
| 19 | 4 | 0.927 | story, watch, time, think, original, feel, people, scene, play, end, plot, work, love, look, director, dont, new, girl, want, character | zombie plenty thistle cheerleader splendid gods gift debut story peter end |
| 20 | 0 | 0.9972 | watch, story, think, time, feel, character, scare, people, family, leave, end, look, ive, want, work, sarah, act, original, juno, build | ve guess director saw end inventive filmmaker work industry james wan brilliantly retro day deliver extremely stylistic visually strike stand tall classics virtually sex gore curse conjuring earn r-rating scare set 1971 conjuring focus marry paranormal researcher ed patrick wilson lorraine vera farmiga warren lecture college interest case theyre think retirement cue perron family parent roger iron livingstone carolyn lili taylor scare live live daughter claim evil rhode island home doesnt warrens discover perrons torment supernatural wanting short conjuring terrify ive seen trying erase torture porn crowd prove difficult director saw doubt finally look pg normally frown junkiest despite sex gore swearing james wants late slap r rate youre wonder frighten think mpa speak behalf day climax middle turn follow doesnt affect conjuring comic relief brilliantly place bring climax chance build fear startle wan understand psychology tension build suspense mere scene construction obviously note exorcist amityville inspiration derive real case file warren famous case date straying ironic style famous cabin woods surface story inventive assure work creative recent memory ive love james wan manage unoriginal haunted-house- possession story cases weave scares pacing sound design camera work describe precise james wan cinematographer john leonetti responsible look insidious fresh visual style unlike include wide shot movement cinematography absolutely stunning rely shaky cam realism leave unique feel separate visually narratively relate toone issue ive recent genre reveal demon insidious sinister example hook story fear unknown filmmaker audience wan definitely learn insidious conjuring apparition arendt reveal audience theyre far focus letting use imagination truly horrifying think filmmakers note mrs wan want complaint isnt wrong market screw trailer conjuring reveal scares avoid trailer command james wants twitter account hard miss tv spots wish fresh mind havenet trailer want stay marketing ultimately overall production value allow conjuring stand rot genre act impressive practical effect perfect classic feel wan time movies must-see summer rating 10let twitter thejoshl think conjuring |
| 21 | 5 | 0.4848 | story, end, character, time, plot, start, scene, watch, fan, director, ride, turn, dead, think, genre, people, work, pretty, enjoy, look | dont summon devil dont priest lucky conjuring preview screen brightest 2013 totally cold trailer story plot turn scary seen conjuring nail-biting hiding-behind-hands youve disappoint paranormal activity insidious likely deliver area failed tell supposedly true story paranormal investigators aim rid family property suspect supernatural visitations disprove turn creaky floorboard slam doors tackle head-on leap faith require buy theme youre okay play pretty genre confines particular incident bring deal describe surround spirit malevolent hide public fairly amityville-like circumstances family new house discover basement sealed boarded-up doorway surprise follows decide look basement surprise james wan manage tire theme haunt possession revive convincingly strange kind truly tension terror enjoy sinister recently equally scary lose bite end conjuring tempo fairly satisfy conclusion turn comical middle minutes fire cylinder head finale intentional lure false sense security work beautifully youre type poo-poo genre general canst suddenly convert believer enjoy classic exorcista amityville poltergeist guarantee conjuring wont disappoint |
| 22 | 2 | 0.5496 | story, dont, character, time, act, work, people, feel, audience, watch, look, home, scene, genre, scary, end, love, director, scare, think | ism avid fan lately ive think isnt scare sinister skin appreciate james wants love saw insidious damn modern ghost story review state kinda lose momentum final act conjuring better scarier tense insidious ism gonna limb years classic rule making bad guy fully audience plus suspense tell throw water bottle screen sheer terror scare finally happened fake jump scares earn r rate blood sex profanity terror build scary masterpiece script acting direction style tone notch wants camera-work far movies choice set stroke genius feel authentic real story set present wouldnt effective scary style making costuming hair style throwback exorcist amityville extremely minor flaws length repetitiveness resolution happen fast dont care general flaws perfect havenet james wants conjuring ive time time scare badly dont miss amaze theater view experienced |
| 23 | 3 | 0.9964 | character, story, people, time, work, dont, scene, love, place, play, set, real, soon, life, feel, plot, violence, dark, want, start | key conjuring freshness evidence ream fan argue internet site new table fan hungry time conjuring right splinter deal admire day age stand tall proud ream remakes sequel teen friendly slasher haunt multiplex frequency days free gore sex automatically alienate portion lustful member fan based serve appetising scare cauldron suspense deliver plenty particular table forget based true story tag kind irrelevant new technological age sell gimmick mean story true play bit regardless hoax charge embellishments buy premises commit scary story director james wan reward compliant story essentially base investigation early seventy paranormal specialist ed lorraine warren aid perron family victim dreadful supernatural event rhode island home wan build deftly let perron family live believe dream home start happen wan build slow instances create palpable sense dread camera work intelligent moment maximum impact yearn warrens involved kill boo-jump scares place care marry superbly mount tension naturally clich convention haunt house strange smells creaky doors ominous cellar supplement wants talented knack scare effective production designs mysterious bruise literal leg pull breath hold game hide seek bona fide pant soil moments conjuring lesson sustain unease finale unleashed script devoid cheese pointless filler refresh sub-genre suffer problems joseph bishara musical score absolute nerve shredder refresh accompaniment doesnt resort telegraph shriek tell afraid overwhelm scene john leonetti cinematography gothic textures house outside lakeside farmhouses strong lead cast vera farmiga patrick wilson lili taylor ron livingston trump met critical box office success conjuring justify reputation superb haunt house |
| 24 | 0 | 0.5532 | watch, story, think, time, feel, character, scare, people, family, leave, end, look, ive, want, work, sarah, act, original, juno, build | conjuring high class hard scare care character story compel feel interest time based true life events ed lorraine warren paranormal investigator set help family terrorize demon terrify case live hadnt share love place 70 agree completely director james wants point view impossible set present example teenage daughter haunt family picture demon diphones post post instagram basically demon turn victim human bad guys isnt perfect thought plot hole scare jump seat build tension let character start craziness set 70 feel 70 slow zoom style wouldnt expect modern believe james wan outdo ill far scary ive life thank james wan luck fast furious d recommend look guess feel audience feel watch exorcist dont look poltergeist time |
| 25 | 6 | 0.992 | story, watch, scene, time, love, people, original, ghost, dont, feel, doesnt, think, creepy, ive, character, remake, end, zombie, want, act | pit human horrendous extra terrestrial end cheap imitation aliens series pitch black stand fine piece sci-fi excellent favorite aspect lighting beautifully employ different colors shade intensity light set mood lend unique feel different normal light generally subject vin diesel bring character life excellent manner skillfully avoid routine portrayal harden criminal riddick diesel character personal journey thankfully vin doesnt drop ball remainder cast exception talented gorgeous claudia black unknown turn marvelous performances animate diverse character unique quirk mannerisms pitch black perfect example resource excessive budget special effect adequate time mean sole focus high budget blockbusters use science fiction medium tell engaging provoke story tell mediocre story use flash science fiction |
| 26 | 5 | 0.9925 | story, end, character, time, plot, start, scene, watch, fan, director, ride, turn, dead, think, genre, people, work, pretty, enjoy, look | pitch black survival story survive hostile alien world hostile enemies task difficult near enemy survive group plot pitch black usual time different variations shine well-written characters group consist different people interest jacks boy secret fryd pilot hard time conscience johns bounty- hunter drug-habit imams holy man face fact god cruel riddick convict murderer learn value characters start live arendt paper genre start care characters reddick feel odd suppose bad guy line light pitch black think character familiar work casting rarely actor fit role radha mitchell suitable fryd cole hauser bring right cruelness sense responsibility johns impressive work vin diesel job riddick hand reddick creepy definitely dangerous deep character suppose survive pitch black friend close enemy closer |
| 27 | 5 | 0.99 | story, end, character, time, plot, start, scene, watch, fan, director, ride, turn, dead, think, genre, people, work, pretty, enjoy, look | admit fan serenity reddick chronicles bite predetermine anticipation flip ism vin geisel fan somewhat mono-dimensional range characters embodiment riddick fit perfectly story play pace encumber gratuitous special effect fit cgi appropriate glare technical defect story start walk breathe air slight concern difference atmosphere microbe common flaw gloss stories canst harp detail radha mitchell smoking hot ridiculous frivolous tough season interplanetary merchant marine ships captain character fry nail perfectly remind andrea start hungarian assassin gilda tigers lady appear drop check heartbeat digress cases trouble grant star pitch black |
| 28 | 5 | 0.9952 | story, end, character, time, plot, start, scene, watch, fan, director, ride, turn, dead, think, genre, people, work, pretty, enjoy, look | doubt excite satisfy ive years plot print banal- ship crash desert planet suns survivor adjust landscape darkness fall monster appear pilot fryd moment cowardice descent atmosphere jettison passengers charge group enlist help convict murderer reddick lead darkness escape ship surgically enhance eye dark simple character complex three-dimensional conflict trait whoas whoas bathe performance uniformly superb radha mitchell fry steel leadership overcome fears prevent bloodshed cole hauser man reddick custody agendum idiosyncratic standards belong vin diesel reddick magnetic screen presence ive years face shadow doesnt deal draw attention draw attention speak moving diesel doesnt trivialise character easily heart gold reddick mean vicious man approach ship let glimpse tentative step care self care technically light effect excellent end spectrum overbright triple sunlight pitch darkness special effect riddick monsters point view add suspense sound effect monster fly ultrasound monster anatomically plausible suitably frightening editing tight jar time literally pad fades time re-orient open shoot end credit wit scene line dialogue single camera shoot important time understand ally caveat science solar system model impossible planets revolve suns vice versa doesnt affect human story havenet point |
| 29 | 1 | 0.9763 | story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind | open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain |
In [49]:
lda_most_representative_docs = get_most_representative_docs(lda_mod_dom_topics_final, n_topics=20)
lda_most_representative_docs.style.applymap(color_green).applymap(make_bold)
Out[49]:
| Probability_Contribution | Topic_Keywords | Most_Representative_Document | |
|---|---|---|---|
| Topic_Number | |||
| 0 | 0.3801 | watch, story, think, time, feel, character, scare, people, family, leave, end, look, ive, want, work, sarah, act, original, juno, build | night wife expect huge fan old school nightmare elm street blood gore dead alive include foreign devil prepare minute heart pump suspense feel kid watch scary scare acting atmosphere creepy face look alive blink moved couldnt sleep night year old year old wow watch caution |
| 1 | 0.5295 | story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind | disgust spend hour raw thats disgust grotesque intrigue canst look act character interest bite time pick scene pointless haze rituals eye glue screen theme human instinct terrify horrid entertain main character turn normal girl different people critique end took love overall definitely worth watch masterpiece break new ground mess disgust entertain runtime |
| 2 | 0.5266 | story, dont, character, time, act, work, people, feel, audience, watch, look, home, scene, genre, scary, end, love, director, scare, think | trick treat night brightest leicester square need round applause end doesnt usually happen uk wasnt fact dougherty thousand tar t undoubtedly seen moment start feel tell fair money chuck set look fantastic straight dvd october theatrical run 2007 unbelievable act effect direction masterful basically set halloweens bunch story interweave clever sort creepshow story isnt standalone time end didnt think bundle surprise litter wonder prom night remake countless saw theatrical run awesome shelve disregarded true masterpiece admission instant classic |
| 3 | 0.6445 | character, story, people, time, work, dont, scene, love, place, play, set, real, soon, life, feel, plot, violence, dark, want, start | having days think prepare near begin scene zero psycho second flat begin 2004 dawn dead wildly chaotic kick-off scene unlike laugh chomp popcorn laugh matter theres violence theres fear tension on-screen violence absolute shock scene scene ordinary people place impossible situation escape time course implacable predator hunt rage virus military attempt maintain control outbreak visit unspeakable innocent people control scale virus severe plan military implement completely plausible action scene masterfully effectively place viewer point view victim crazed scarily human zombies portrayal violence pull punches people age group walk life destroy remorse attempt soft-pedal fragility human life earth vulnerability right nasty virus thought stay youve leave theater add nice taste fear soundtracks amaze convey tension dread sadness scenes story fairly tight complaint act soldiers didnt feel authentic reason overall id zombie ive seen fact effective thriller ive seen |
| 4 | 0.5072 | story, watch, time, think, original, feel, people, scene, play, end, plot, work, love, look, director, dont, new, girl, want, character | nattevagten thriller edge seat saw right begin original story suck doesnt let end thrillers grip extremely rare time people guess twist saw james wan leigh whannell creative head project set new standards think youre hard-boiled think watch saw creep surprise expectations saw advertise new se7en definitely traditions saw job creepy jigsaw gruesome killer cinema looking time wan channel monster peer drift ridiculousness establish villain superhuman evil saw near trap cop depict stupid killer unrealistically smart outshine opponent perfect plans hey se7en silence lambs didnt care realism saw flaws sped-up track shot time structure script weird jump period time characters line bite clichéd consider day independent filmmakers literally budget inappropriate petty technical subtleties wan channel original stir canst remember time ive surprise movies final twist saw end didnt thriller original piece independent film-making cube ism look forward wan whannell career develop fine sleeper |
| 5 | 0.4848 | story, end, character, time, plot, start, scene, watch, fan, director, ride, turn, dead, think, genre, people, work, pretty, enjoy, look | dont summon devil dont priest lucky conjuring preview screen brightest 2013 totally cold trailer story plot turn scary seen conjuring nail-biting hiding-behind-hands youve disappoint paranormal activity insidious likely deliver area failed tell supposedly true story paranormal investigators aim rid family property suspect supernatural visitations disprove turn creaky floorboard slam doors tackle head-on leap faith require buy theme youre okay play pretty genre confines particular incident bring deal describe surround spirit malevolent hide public fairly amityville-like circumstances family new house discover basement sealed boarded-up doorway surprise follows decide look basement surprise james wan manage tire theme haunt possession revive convincingly strange kind truly tension terror enjoy sinister recently equally scary lose bite end conjuring tempo fairly satisfy conclusion turn comical middle minutes fire cylinder head finale intentional lure false sense security work beautifully youre type poo-poo genre general canst suddenly convert believer enjoy classic exorcista amityville poltergeist guarantee conjuring wont disappoint |
| 6 | 0.5212 | story, watch, scene, time, love, people, original, ghost, dont, feel, doesnt, think, creepy, ive, character, remake, end, zombie, want, act | shutters begin thun young photographer girlfriend jane accidentally run young woman drive home decide leave dead victim drive thun discover strange mysterious shadow appear photo think thats bad picture realize sinister shadow-both tun jane plague extremely unsettle dreams unable cope start investigate phenomenon ghost spirit photography lead discovery tunes past possible clue identity ghostly nemesis shutter hit thailand easy notice whit offer truly eerie moment share book scare tooth final image scream act surprisingly fascinate watched look |
| 7 | 0.4681 | story, character, time, feel, dont, scene, people, think, look, watch, doesnt, work, scare, director, dark, right, play, scary, house, end | herems theater sport gorgeous black white photography farsis vampire art-house flick apparently unstoppable hype machine power vice kino lorber usually distributor worth watching promote picture actual appeal dishonestly pace trailer ingredient sound amazing perfect bait theres market hallmark different turn mean yes farsis culturally void iranian bakersfield california bakersfield starkly cinematics endeavor feel extremely western youre catch glimpse under- represent culture youre cool role farsi speak american actors mean serve place inside persian community bakersfield interest thats cases fictional western-pulp style iran community bad town set series creative decision cool factor deep purpose yes art house flick ambitious kind celebrate style substance assume disaffection equal cool choose aesthetic principle theme turn stab greatness imitate giant nouvelle vague post punk wave cinema cite spaghetti westerns influence thats soundtracks image music unsupported character theme miss place yes vampire lead affect beautiful cool vampire narrative soon jarmusch lovers left alive jarmusch season filmmaker able non-narrative attempts place emphasis style character theme lovers left alive lovers music video tonal exercise girl walks home night isn t doesnt aspire scary creepy obviously doesnt ism fine tone dont expect day scary vampire movieland leave gorgeous photography lyle vincent frankly save flick intolerable bear feature strong cinematic moments shoot truly joy rely completely love beautiful cinematography thats save grace bright spots music excellent rise height sequence suppose supporting unquestionably director use music affect nice moment sprinkle final climactic death beautifully piece filmmaking lead lovely fun watch scene end tedious meditative pretty obvious short stretch means love cinema slow contemplate languid shoot real hold underneath image music feel mean exercise ism fresh different meaning surface barely qualifies theres new unique tired snoop amirpour future hope aesthetic day engine purpose cinema unique |
In [50]:
lda_topic_dist = get_topic_distribution(lda_mod_dom_topics_final, lda_most_representative_docs, n_topics=20)
lda_topic_dist.style.applymap(color_green).applymap(make_bold)
Out[50]:
| Documents_per_Topic | Percent_of_Total_Corpus | Topic_Keywords | |
|---|---|---|---|
| Topic_Number | |||
| 0 | 140 | 0.14 | watch, story, think, time, feel, character, scare, people, family, leave, end, look, ive, want, work, sarah, act, original, juno, build |
| 1 | 128 | 0.128 | story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind |
| 2 | 112 | 0.112 | story, dont, character, time, act, work, people, feel, audience, watch, look, home, scene, genre, scary, end, love, director, scare, think |
| 3 | 64 | 0.064 | character, story, people, time, work, dont, scene, love, place, play, set, real, soon, life, feel, plot, violence, dark, want, start |
| 4 | 160 | 0.16 | story, watch, time, think, original, feel, people, scene, play, end, plot, work, love, look, director, dont, new, girl, want, character |
| 5 | 92 | 0.092 | story, end, character, time, plot, start, scene, watch, fan, director, ride, turn, dead, think, genre, people, work, pretty, enjoy, look |
| 6 | 120 | 0.12 | story, watch, scene, time, love, people, original, ghost, dont, feel, doesnt, think, creepy, ive, character, remake, end, zombie, want, act |
| 7 | 184 | 0.184 | story, character, time, feel, dont, scene, people, think, look, watch, doesnt, work, scare, director, dark, right, play, scary, house, end |
In [51]:
prepared = pyLDAvis.gensim.prepare(lda_mod, corpus, dictionary)
display(pyLDAvis.display(prepared))
LDA Mallet¶
Cf. http://mallet.cs.umass.edu/¶
In [52]:
#Cf. https://github.com/RaRe-Technologies/gensim/issues/2137 (for fix)
#File dl: http://mallet.cs.umass.edu/dist/mallet-2.0.8.zip
os.environ['MALLET_HOME'] = '.\\mallet-2.0.8\\'
mallet_path = '.\\mallet-2.0.8\\bin\\mallet'
In [53]:
mod_list_ldamallet_umass, coh_list_ldamallet_umass = compute_coherence_values(LdaMallet, dictionary, all_reviews_text, max_topics=20,
corpus=corpus, topics=False, transformed_vectorizer=False, tfidf_norm=False,
min_topics=2, stride=3, n_top_words=False, measure='u_mass',
mallet_flag=True, mallet_path=mallet_path)
mod_list_ldamallet_cv, coh_list_ldamallet_cv = compute_coherence_values(LdaMallet, dictionary, all_reviews_text, max_topics=20,
corpus=corpus, topics=False, transformed_vectorizer=False, tfidf_norm=False,
min_topics=2, stride=3, n_top_words=False, measure='c_v',
mallet_flag=True, mallet_path=mallet_path)
Coherence of model with 2 topics has been computed Coherence of model with 5 topics has been computed Coherence of model with 8 topics has been computed Coherence of model with 11 topics has been computed Coherence of model with 14 topics has been computed Coherence of model with 17 topics has been computed All coherence values have been computed for the <class 'gensim.models.wrappers.ldamallet.LdaMallet'> using the U_MASS measure Coherence of model with 2 topics has been computed Coherence of model with 5 topics has been computed Coherence of model with 8 topics has been computed Coherence of model with 11 topics has been computed Coherence of model with 14 topics has been computed Coherence of model with 17 topics has been computed All coherence values have been computed for the <class 'gensim.models.wrappers.ldamallet.LdaMallet'> using the C_V measure
In [54]:
best_num_ldamallet_topics = show_best_num_topics('LDA_Mallet', coh_list_ldamallet_umass, coh_list_ldamallet_cv, max_topics=20, min_topics=2, stride=3)
print('The most coherent number of LDA_Mallet topics to use is: {}'.format(best_num_ldamallet_topics))
The most coherent number of LDA_Mallet topics using the U_MASS measure is: 2 The most coherent number of LDA_Mallet topics using the C_V measure is: 8 The most coherent number of LDA_Mallet topics to use is: 8
In [55]:
lda_mallet = LdaMallet(mallet_path, corpus=corpus, num_topics=best_num_ldamallet_topics, id2word=dictionary)
In [60]:
#topics_matrix = lda_mallet.show_topics(formatted=True, num_words=20)
#topics_matrix = np.array(topics_matrix)
#topics_matrix.shape
In [61]:
#topic_words = topics_matrix[:,1]
#for i in topic_words:
# print([reg_ex.sub('', w) for w in i.split() if len(reg_ex.sub('', w)) > 0])
# print('\n')
In [56]:
lda_mallet_df = get_topics(lda_mallet, best_num_ldamallet_topics)
#print(lda_mallet_df)
lda_mallet_df
Out[56]:
| Topic # 01 | Topic # 02 | Topic # 03 | Topic # 04 | Topic # 05 | Topic # 06 | Topic # 07 | Topic # 08 | |
|---|---|---|---|---|---|---|---|---|
| 0 | story | house | scene | watch | scene | performance | character | time |
| 1 | ghost | plot | dark | original | vampire | actor | people | feel |
| 2 | love | scare | story | fan | shoot | creepy | dont | leave |
| 3 | end | story | work | ism | girl | story | sound | doesnt |
| 4 | scary | genre | ring | dont | work | room | live | character |
| 5 | play | family | psychological | youre | days | years | thriller | story |
| 6 | short | audience | twist | remake | night | release | kill | dont |
| 7 | world | haunt | excellent | minute | town | time | love | watch |
| 8 | time | true | water | flick | woman | year | killer | sense |
| 9 | start | add | tale | ive | nature | night | play | start |
| 10 | creepy | doesnt | performance | dead | guy | young | end | happen |
| 11 | work | act | fact | people | plot | genre | feel | end |
| 12 | suspense | feel | slowly | enjoy | life | director | situation | didnt |
| 13 | bite | music | american | bad | zombie | play | place | friend |
| 14 | funny | sinister | japanese | movies | home | stand | time | real |
| 15 | moment | day | brilliant | classic | experience | treat | world | understand |
| 16 | hollywood | conjuring | revenge | time | wife | fan | school | scary |
| 17 | atmosphere | eye | viewer | wasnt | shadow | set | problem | people |
| 18 | genre | element | perfect | love | real | title | young | characters |
| 19 | canst | character | pretty | expect | follow | cinematography | high | hard |
In [57]:
ldamallet_hellinger_topic_distances = get_most_similar_topics(lda_mallet, lda_mallet_df, num_topics=best_num_ldamallet_topics, mallet_flag=True)
pretty_print(ldamallet_hellinger_topic_distances)
Hellinger Distances between topics are: ______________________________________________________________________ (0.9545340440576985, 'Topic # 04', 'Topic # 05') (0.9426037977006119, 'Topic # 03', 'Topic # 04') (0.9359234608592298, 'Topic # 02', 'Topic # 07') (0.9341593554859184, 'Topic # 04', 'Topic # 06') (0.9340884820207142, 'Topic # 01', 'Topic # 04') (0.9316609666635765, 'Topic # 06', 'Topic # 08') (0.9258880611682975, 'Topic # 05', 'Topic # 06') (0.9247568336998111, 'Topic # 03', 'Topic # 07') (0.924315383410101, 'Topic # 05', 'Topic # 08') (0.9237283410570338, 'Topic # 04', 'Topic # 07') (0.9218963025154837, 'Topic # 02', 'Topic # 05') (0.9215309384100554, 'Topic # 06', 'Topic # 07') (0.9189364773675167, 'Topic # 04', 'Topic # 08') (0.9178189761080324, 'Topic # 02', 'Topic # 04') (0.9145282089633445, 'Topic # 03', 'Topic # 05') (0.9108689923818001, 'Topic # 05', 'Topic # 07') (0.9091161059483536, 'Topic # 01', 'Topic # 03') (0.9087474646722671, 'Topic # 01', 'Topic # 05') (0.9087000354313345, 'Topic # 01', 'Topic # 07') (0.907448147217302, 'Topic # 03', 'Topic # 06') (0.9056529464165876, 'Topic # 01', 'Topic # 06') (0.9049301973905479, 'Topic # 02', 'Topic # 03') (0.9039229616814698, 'Topic # 02', 'Topic # 06') (0.900690730724466, 'Topic # 01', 'Topic # 02') (0.8923821813919287, 'Topic # 07', 'Topic # 08') (0.8868980857601233, 'Topic # 01', 'Topic # 08') (0.8863877592324878, 'Topic # 03', 'Topic # 08') (0.8783005976699784, 'Topic # 02', 'Topic # 08') ______________________________________________________________________
In [58]:
lda_mallet_dom_topics_df, lda_mallet_dom_topics_final = get_dominant_topics(lda_mallet, corpus, all_reviews_joined)
In [59]:
#lda_mallet_dom_topics_final.style.applymap(color_green).applymap(make_bold)
lda_mallet_dom_topics_final_first30 = lda_mallet_dom_topics_final.head(30)
lda_mallet_dom_topics_final_first30.style.applymap(color_green).applymap(make_bold)
Out[59]:
| Dominant_Topic | Probability_Contribution | Topic_Keywords | Original_Text | |
|---|---|---|---|---|
| Document_Number | ||||
| 0 | 1 | 0.2769 | house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character | coherent plot tell non-straightforward fashion open interpretations leave require audience fill-in open end possibly filling-in require viewings considerable think sense watch wailing time impression kind watch ponder read watch attempt people interpret explain different conclusion coherent underlie plot matt shift piece puzzle attempt recreate coherent narrative piece fit incompetence underlie story possibly intentionally possibly fundamentally inconsistent optical illusion escher drawing appear describe physical object fact dont physical sense accordingly enjoy boil content technically well-made cinematography acting costumes plot sense tell deceptive lure think plot sense matt sufficient thought leave lingering irritate feel dissatisfaction confusion maybe precisely point tell story purpose instill audience feel confusion face sequence event sense life times |
| 1 | 2 | 0.2751 | scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty | spoiler review extraordinary hour technically-perfect humanistic fine writer directors business hauteur saw devil cipher close analog suggest david lynches 2001 mulholland drive technically perfect humanistic suspense opus captivate length walk theatre shake head wonder saw film-makers understand secret story tell present story viewer feel share experience protagonist tell story viewer going provide explanation wailing means re-quote writer director numerous interview enigmatically begin wonder nature god -- recommended levels entertainment puzzled clinic engage wont let |
| 2 | 7 | 0.4288 | time, feel, leave, doesnt, character, story, dont, watch, sense, start, happen, end, didnt, friend, real, understand, scary, people, characters, hard | wailing open quote bible easy forget fact watch certain point clear purpose quote served warning religious person scare wailing work religious parables overcome lengthy run time tonal imbalanced deliver funny truly terrify mysterious strange small village south korean village plague sickness police think wild mushroom blame police officer jong-goo think stranger meet woman information mysterious man slowly begin fall rabbit hole consume life daughter start sign sickness personal figure trust avoid director hong-jin nap struggle tonal balance time pitch perfect time place tone struggle stay consistent primarily funny brief burst depravity violence comedy work laugh numb times horrific imagery sudden effective balance lose act comedy slapstick fit rest character over-the-top act exaggerate different introduced middle section moment unintentional humor example man strike lighten caller id shaman phone shaman fortunately balance act smartly switch act intense terrify moment comedy offset terrify act start unfold audience gain new appreciation rest start reinterpret certain scenes problem scene sense ended feel natural fit story progressed require end sense hour minutes middle portion resolved shaman screen time perform ritual end important zombie scene feel awkward didnt fit rest want zombie grow popularity scene bring numb random character serve purpose rest despite scene previously mention overacting act generally speak fantastic father task solve happen daughter convey terror hatred build intense persona carry priest train advice father equally effective bring concern innocent quality terror ultimately happen young daughter sick shines channeling inner linda blair emphatically deliver horrible dirty line child performance truly terrify watch hatred eye slowly end religious overtone clearer locusts attack individual white black suggest characters true nature scene truly shines slowly unveil real nature certain characters minute change perspective things minute switch determine trust reveal realize detail setup reveal twist lead ultimately putt faith wrong people wailing 2016 directed hong-jin na screenplay hong-jin na starring woo-hee jeong-min hwang so-yeon range won krak run time hour minute |
| 3 | 7 | 0.1803 | time, feel, leave, doesnt, character, story, dont, watch, sense, start, happen, end, didnt, friend, real, understand, scary, people, characters, hard | hell ride minute stop look time leave relieved left thats engage interest story watch read review smorgasbord genre positive review doesnt feel means yes storytelling fluid write multiple genre aspect didnt feel place works work feel redundant point genres similar witch necessarily term storyline pace severity terror psychological wailing deliver somewhat complex storyline cinematography breathtaking jun kunimura ridiculous oh-so-good actor point ism point mention asian technically guess lol dont think necessary pigeonhole |
| 4 | 1 | 0.2393 | house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character | want start review completely jump scares play integral movies rely support story leave displeased wailing special jump scare instead viewers story rich mystery character struggles dark atmosphere design amazingly craft scene blood run cold addition carry subtext leave viewer question evidence interpret feel director time research real experience possible wailing bite bore slow burner construct modern running time th 36min patient man pay end genre dead look cash grab mainstream wide release hide gems |
| 5 | 4 | 0.3068 | scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow | ism biased set city live work oxford street piccadilly circus pass morning usually teem crowd people completely send shiver spine usually watch locate nondescript midwestern village easy detach event unfold screen occur place home entirely new sense reality previously unaccustomed judge days entirely merit easy arrive conclusion fantastic achievements coming-of-age sort director danny boyle canst mtv-inspired vanity beach self-consciously trendy posture trainspotting appeal shame initially expect days similar treatment thankfully fear prove unfounded discard straight open sequence effortless fearsome rest joy terrify joy joy nonetheless true minimalism effective overblown bravado definitely true scene complete silence entire metropolis grainy picture serve add documentary-style quality situation real bear definitely wise choice digital video occasionally meet people think days wasnt scary gory people tell 2001 boring memento confusing ignore didnt understand purpose confuse change pace feel distract personally regard important reviewer point succinct point scarier end world world repopulate maniacs think real days lies 28 days romero zombie flick yore ultimately allegory day live age constantly confront violence medium ape right start violence beget violence humanity face uncertain future applaud danny boilers bravery days undoubtedly commercial risk majority cinema-going public prefer escapism word caution remember rage human-made disease allegory masterpiece time days misunderstand considerable people doubt history classic perfectly sum confuse era live didnt advisable days chance haunt experience look right angle danny boyle year leave hope hell continue |
| 6 | 4 | 0.3941 | scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow | virus rage virus infect person mad extreme rage hungry blood day outbreak london cause entire britain dead evacuate leave blood-thirsty infect population handful solitary normal persons civilization halt society destroy limit survivor fight existence frequent attack vicious victims sounds familiar 28 days classic horde zombie biohazard movies touch art danny boyle able bring focus extensive special-effects-controlled gory action sequence infect normals heavy background music theres tinge sadness emptiness helplessness consider london scene background music theres electrify action gore describe picture life condition talk mistake loophole couldnt weaken tight-gripping storyline achievement viewer stay neutral place virus talk metaphor rage social disease flick person harsh critic zombie watch stars mention billian murphy awesome |
| 7 | 3 | 0.1579 | watch, original, fan, ism, dont, youre, remake, minute, flick, ive, dead, people, enjoy, bad, movies, classic, time, wasnt, love, expect | ism amaze negative review think succeed level artistic atmospheric pace sympathetic characters fantastic climax music nicely eerie open scene london street worth price admission ism stubborn viewer normally benefit early critical buzz manner excuse time ism completely impressed incidentally think interest day inspire knockoffs knockoffs 28 days apparently owe john wyndham classic disaster novel day triffids thats things highly recommended |
| 8 | 4 | 0.4505 | scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow | 2003 state-side release danny boilers 28 days advertise chockful scare-fest didnt day ago gotta feel somewhat embellishment promoters environmental terrorist attack lab contain diseased chimp infect rage virus unwittingly let loose plague lay waste england rest society days title cut abandon london coma-tized bicycle courier jim cillian murphy effective wake stasis hospital search london sign life rescue rage zombie couple survivalists lovely naomi harris follow place place alive meet man daughter brendan gleeson terrific megan burns refuge london-town record message paradise salvation track frank man shortwave radio feel meditation happen people reduce low elements friend tell movies run zombie inspire zombie remake dawn dead dawn pretty full-throttle action hybrid start finish play emphasis creature lucky survivors disturb lesson nature survival interest disturb flick sell wrong |
| 9 | 4 | 0.4439 | scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow | key sci-fi genre alive cinemas material technique filmmakers present competent average creative havenet havenet style form george romero prime era film-making bring forth memorable trilogy time genre consider romero list director included important source independent 1968 inventive result masterpiece lacklusters director danny boyle author alex garland cook yarn similar romero wouldnt satisfying theyve essential success days later- idea practice years turn fresh audience feel repelled excited terrified nauseous enthralled wont leave feel weave complete hack work boyle team set freshen zombie string precise term zombie movie- hear living-dead utter hear infected new word people rage infect britain monkey virus let loose island begin infectious spread cut man jim lie hospital bed wander abandon street view tear fragment society boyle implement atmosphere heavily action chase scene indication dedication details jim soon survivors include selena naomie harris father daughter brendan gleeson megan burns hear salvation radio decide brave military outpost thats true salvation outside military typos boyle producer andrew macdonald spellbind thats proper terminology adaptation trainspotting craft par arguably toppings story characters captivate care plight jim companions photography anthony dod mantle striking digital photography blair witch use non-professional camera equipment add proper shade needed shoot composition different edit chris gill quick expect attack scenes fast infect throw blood scream on-looker new infect transform seconds result follow mark tildesley production designs john murphys music evoke haunting evocative mood mundane scenes acting consider actor well-known believable script ism days cup tea science fiction fan immediately hear preview tv add dismiss phooey rubble borrow video-store point view similarity romero military scene remind day dead chain zombie practical reasons supermarket scene unneeded consider satirical reverence dawn dead understand boyle isnt 100 original point genres history credible job different tone different set country different envelop view scene structures overall days construct execute sci-fi youve seen sci-fi youve combined sense modern audience fascinate |
| 10 | 2 | 0.3339 | scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty | let right vampire seen smarter scary nuanced doesnt feel thriller feel literature detail bizarre misadventure pair pre-teen star cross lovers androgynous vampired phenomenal regard detail young oskar kare hedebrant life spot stuck incredibly painful period post-childhood pre-adolescence oskar aware girls idea contend small age brutalize boy result terribly collect news clip violent crime let rage day strange young girl eli klina leandersson appear playground fast friend begin look oskar eli innocently spend night occasion privy oskar doesnt happens elias caretaker serial killer brutal order desanguinating victim bucket soon oskar realize new friend bite tragedy shock violence eli leave fend desperately stave urge drink fresh blood form delicate new bond oskar remake let right dont wait hope capture magic original gorgeously shot impeccably written fully realize unmatched respect vampire lore acting producer amaze young actors odd recapture brilliant melancholy chemistry astronomical let right think traditional opt endless ultra violence cut favor kiddie friendly rating director tomas alfredson steer line right middle violence brutal horrific dwell leave question kidney display stillness mimic oskar stage stuck moving hope growth begin changes apparent stillness oskar elik oskar grow change man eli stick burdensome fate quiet understate ending haunting whimsical forth times people understanding kind people beg dont miss time silence lambs real chance home oscar gold deserve a+ |
| 11 | 2 | 0.4811 | scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty | let right heart sweet coming-of-age story unique different defy categorizations swedish adapt john aside lindqvist bestselling book director tomas alfredson dare mix pleasure pain horrify tender let right storyline reveal secret early leave surprise necessarily rare review plot better 12-year-olds oskar kare hedebrant eli klina leandersson meet snowy afternoon jungle gym courtyard oskar house complex outside stockholm young tender attraction apparent right start think relationship headed deep dark secret discover reveal audience repulse curiously fascinate time similar fashion yellow crime scene tape bring close warn support cast completely beholden narrative revolve adorable young couple performance rival ive actor age innocence vulnerability hedebrant oskar tour-de-force admirably carry shoulders leandersson match scene scene line line result literally chills production value stellary technical aspect -- lighting original music johan soderqvist hoyle van hoytema cinematography -- combine perfect synchronization produce hitchcockian tale bring love light dark drama imaginable let right overwhelm choice best narrative feature north american premiere 2008 tribes festival truly well-deserved honor tomas alfredson craft brilliant work art leave shake head wonder |
| 12 | 2 | 0.2358 | scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty | tomas alfredson let right original dark twist gory fantasy special hard classify merely exercise style brilliant piece amoral storytellings characters action defy logic common sense dont wanna spoil moment ll mean revenge moment story happens remind fantasy tale oskar kåre hedebrant year-old bully boy befriend develop innocent crush new neighbor eli klina leandersson happen vampired twist tale revenge pubescent love visual flair swim pool scene classic creative direct impressive performance young pair protagonists hollywood course didnt waste time announce upcoming remake lazy read subtitles likely remake turn pg-13 order money fill moral value prudish parent let kid watch dont vampired year ago ignore future bomb enjoy original youre treaty |
| 13 | 2 | 0.489 | scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty | people review idb focus sweet friendship year old human vampire leads huge element sweet story childhood friendship godfather story father concern worry kids true descriptions miss point think seen truth let right dark measure brilliant story tell week ago disturb echo stay refuse enjoy prior knowledge impossible review properly disclose plot elements warn spoilers ahead comment said character hokan key story end lead believe oskar step place fact reveal final scene dark nightmarish core blossom friendship witness prior offer redemption oskar path relationship eli takes håkan butcher boy old oskar food elik story storytelling high order dark whisper hint suggest questions allude play mind answered hokan pick victim presumably elias preference elias interest oskar elias true nature time briefly glimpse true physical age fact late middle age roughly age håkan gift oskar moments brutal slaughter child tormentors entranced glamour oskar happy happy happily sunset oskar presumably deaden soul destroy husk person hokan butcher fellow human provide eli food puzzle people think sweet fact conclusion utterly chilling oskar happily swap commonplace misery childhood bully fate truly hell earth happily smile shoot doesnt clue superlative heartily recommends tale succeed suggestion play imagination truly addition vampire cannon variation bram stokers brilliant original |
| 14 | 3 | 0.1925 | watch, original, fan, ism, dont, youre, remake, minute, flick, ive, dead, people, enjoy, bad, movies, classic, time, wasnt, love, expect | fond vampire genre artistic poetic profound explore nature evil world child pure evil pure goodness somewhat discernible achieve astound array contrast allow evil coexist thought-provoking leave audience yearn read novel short gems need remake accord dont need wait 2010 watch |
| 15 | 3 | 0.155 | watch, original, fan, ism, dont, youre, remake, minute, flick, ive, dead, people, enjoy, bad, movies, classic, time, wasnt, love, expect | zombies zombies cheap outta original idea moves mandatory width train play beautiful girl sucker long-haired woman short skirts babe morality story ethic means played |
| 16 | 4 | 0.1739 | scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow | different zombies army undead create industrial mishap result overwhelm chaos danger address morality root cause personally love pretty woman enjoy leg beautiful hair girl hat |
| 17 | 5 | 0.1556 | performance, actor, creepy, story, room, years, release, time, year, night, young, genre, director, play, stand, treat, fan, set, title, cinematography | train susan treat mind eyes look capitalism vs humanity man vs empathy remember time enjoy foreign act excellent effeminate young man story original definitely eye delicious cheerleader |
| 18 | 3 | 0.2029 | watch, original, fan, ism, dont, youre, remake, minute, flick, ive, dead, people, enjoy, bad, movies, classic, time, wasnt, love, expect | dont youths fun action-packed dangerous elements innovative pretty girl importantly successful yup hollywood remake entry series redundant unnecessary inferior whitewash woman western taliban style ugly hollywood remakes paul fig jar jar abrams negotiate cut speak |
| 19 | 0 | 0.1371 | story, ghost, love, end, scary, play, short, world, time, start, creepy, work, suspense, bite, funny, moment, hollywood, atmosphere, genre, canst | zombie plenty thistle cheerleader splendid gods gift debut story peter end |
| 20 | 1 | 0.6055 | house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character | ve guess director saw end inventive filmmaker work industry james wan brilliantly retro day deliver extremely stylistic visually strike stand tall classics virtually sex gore curse conjuring earn r-rating scare set 1971 conjuring focus marry paranormal researcher ed patrick wilson lorraine vera farmiga warren lecture college interest case theyre think retirement cue perron family parent roger iron livingstone carolyn lili taylor scare live live daughter claim evil rhode island home doesnt warrens discover perrons torment supernatural wanting short conjuring terrify ive seen trying erase torture porn crowd prove difficult director saw doubt finally look pg normally frown junkiest despite sex gore swearing james wants late slap r rate youre wonder frighten think mpa speak behalf day climax middle turn follow doesnt affect conjuring comic relief brilliantly place bring climax chance build fear startle wan understand psychology tension build suspense mere scene construction obviously note exorcist amityville inspiration derive real case file warren famous case date straying ironic style famous cabin woods surface story inventive assure work creative recent memory ive love james wan manage unoriginal haunted-house- possession story cases weave scares pacing sound design camera work describe precise james wan cinematographer john leonetti responsible look insidious fresh visual style unlike include wide shot movement cinematography absolutely stunning rely shaky cam realism leave unique feel separate visually narratively relate toone issue ive recent genre reveal demon insidious sinister example hook story fear unknown filmmaker audience wan definitely learn insidious conjuring apparition arendt reveal audience theyre far focus letting use imagination truly horrifying think filmmakers note mrs wan want complaint isnt wrong market screw trailer conjuring reveal scares avoid trailer command james wants twitter account hard miss tv spots wish fresh mind havenet trailer want stay marketing ultimately overall production value allow conjuring stand rot genre act impressive practical effect perfect classic feel wan time movies must-see summer rating 10let twitter thejoshl think conjuring |
| 21 | 1 | 0.3957 | house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character | dont summon devil dont priest lucky conjuring preview screen brightest 2013 totally cold trailer story plot turn scary seen conjuring nail-biting hiding-behind-hands youve disappoint paranormal activity insidious likely deliver area failed tell supposedly true story paranormal investigators aim rid family property suspect supernatural visitations disprove turn creaky floorboard slam doors tackle head-on leap faith require buy theme youre okay play pretty genre confines particular incident bring deal describe surround spirit malevolent hide public fairly amityville-like circumstances family new house discover basement sealed boarded-up doorway surprise follows decide look basement surprise james wan manage tire theme haunt possession revive convincingly strange kind truly tension terror enjoy sinister recently equally scary lose bite end conjuring tempo fairly satisfy conclusion turn comical middle minutes fire cylinder head finale intentional lure false sense security work beautifully youre type poo-poo genre general canst suddenly convert believer enjoy classic exorcista amityville poltergeist guarantee conjuring wont disappoint |
| 22 | 1 | 0.2305 | house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character | ism avid fan lately ive think isnt scare sinister skin appreciate james wants love saw insidious damn modern ghost story review state kinda lose momentum final act conjuring better scarier tense insidious ism gonna limb years classic rule making bad guy fully audience plus suspense tell throw water bottle screen sheer terror scare finally happened fake jump scares earn r rate blood sex profanity terror build scary masterpiece script acting direction style tone notch wants camera-work far movies choice set stroke genius feel authentic real story set present wouldnt effective scary style making costuming hair style throwback exorcist amityville extremely minor flaws length repetitiveness resolution happen fast dont care general flaws perfect havenet james wants conjuring ive time time scare badly dont miss amaze theater view experienced |
| 23 | 1 | 0.4168 | house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character | key conjuring freshness evidence ream fan argue internet site new table fan hungry time conjuring right splinter deal admire day age stand tall proud ream remakes sequel teen friendly slasher haunt multiplex frequency days free gore sex automatically alienate portion lustful member fan based serve appetising scare cauldron suspense deliver plenty particular table forget based true story tag kind irrelevant new technological age sell gimmick mean story true play bit regardless hoax charge embellishments buy premises commit scary story director james wan reward compliant story essentially base investigation early seventy paranormal specialist ed lorraine warren aid perron family victim dreadful supernatural event rhode island home wan build deftly let perron family live believe dream home start happen wan build slow instances create palpable sense dread camera work intelligent moment maximum impact yearn warrens involved kill boo-jump scares place care marry superbly mount tension naturally clich convention haunt house strange smells creaky doors ominous cellar supplement wants talented knack scare effective production designs mysterious bruise literal leg pull breath hold game hide seek bona fide pant soil moments conjuring lesson sustain unease finale unleashed script devoid cheese pointless filler refresh sub-genre suffer problems joseph bishara musical score absolute nerve shredder refresh accompaniment doesnt resort telegraph shriek tell afraid overwhelm scene john leonetti cinematography gothic textures house outside lakeside farmhouses strong lead cast vera farmiga patrick wilson lili taylor ron livingston trump met critical box office success conjuring justify reputation superb haunt house |
| 24 | 1 | 0.3826 | house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character | conjuring high class hard scare care character story compel feel interest time based true life events ed lorraine warren paranormal investigator set help family terrorize demon terrify case live hadnt share love place 70 agree completely director james wants point view impossible set present example teenage daughter haunt family picture demon diphones post post instagram basically demon turn victim human bad guys isnt perfect thought plot hole scare jump seat build tension let character start craziness set 70 feel 70 slow zoom style wouldnt expect modern believe james wan outdo ill far scary ive life thank james wan luck fast furious d recommend look guess feel audience feel watch exorcist dont look poltergeist time |
| 25 | 2 | 0.3585 | scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty | pit human horrendous extra terrestrial end cheap imitation aliens series pitch black stand fine piece sci-fi excellent favorite aspect lighting beautifully employ different colors shade intensity light set mood lend unique feel different normal light generally subject vin diesel bring character life excellent manner skillfully avoid routine portrayal harden criminal riddick diesel character personal journey thankfully vin doesnt drop ball remainder cast exception talented gorgeous claudia black unknown turn marvelous performances animate diverse character unique quirk mannerisms pitch black perfect example resource excessive budget special effect adequate time mean sole focus high budget blockbusters use science fiction medium tell engaging provoke story tell mediocre story use flash science fiction |
| 26 | 2 | 0.2874 | scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty | pitch black survival story survive hostile alien world hostile enemies task difficult near enemy survive group plot pitch black usual time different variations shine well-written characters group consist different people interest jacks boy secret fryd pilot hard time conscience johns bounty- hunter drug-habit imams holy man face fact god cruel riddick convict murderer learn value characters start live arendt paper genre start care characters reddick feel odd suppose bad guy line light pitch black think character familiar work casting rarely actor fit role radha mitchell suitable fryd cole hauser bring right cruelness sense responsibility johns impressive work vin diesel job riddick hand reddick creepy definitely dangerous deep character suppose survive pitch black friend close enemy closer |
| 27 | 0 | 0.1951 | story, ghost, love, end, scary, play, short, world, time, start, creepy, work, suspense, bite, funny, moment, hollywood, atmosphere, genre, canst | admit fan serenity reddick chronicles bite predetermine anticipation flip ism vin geisel fan somewhat mono-dimensional range characters embodiment riddick fit perfectly story play pace encumber gratuitous special effect fit cgi appropriate glare technical defect story start walk breathe air slight concern difference atmosphere microbe common flaw gloss stories canst harp detail radha mitchell smoking hot ridiculous frivolous tough season interplanetary merchant marine ships captain character fry nail perfectly remind andrea start hungarian assassin gilda tigers lady appear drop check heartbeat digress cases trouble grant star pitch black |
| 28 | 2 | 0.3035 | scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty | doubt excite satisfy ive years plot print banal- ship crash desert planet suns survivor adjust landscape darkness fall monster appear pilot fryd moment cowardice descent atmosphere jettison passengers charge group enlist help convict murderer reddick lead darkness escape ship surgically enhance eye dark simple character complex three-dimensional conflict trait whoas whoas bathe performance uniformly superb radha mitchell fry steel leadership overcome fears prevent bloodshed cole hauser man reddick custody agendum idiosyncratic standards belong vin diesel reddick magnetic screen presence ive years face shadow doesnt deal draw attention draw attention speak moving diesel doesnt trivialise character easily heart gold reddick mean vicious man approach ship let glimpse tentative step care self care technically light effect excellent end spectrum overbright triple sunlight pitch darkness special effect riddick monsters point view add suspense sound effect monster fly ultrasound monster anatomically plausible suitably frightening editing tight jar time literally pad fades time re-orient open shoot end credit wit scene line dialogue single camera shoot important time understand ally caveat science solar system model impossible planets revolve suns vice versa doesnt affect human story havenet point |
| 29 | 2 | 0.1814 | scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty | open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain |
In [60]:
lda_mallet_most_representative_docs = get_most_representative_docs(lda_mallet_dom_topics_final, n_topics=20)
lda_mallet_most_representative_docs.style.applymap(color_green).applymap(make_bold)
Out[60]:
| Probability_Contribution | Topic_Keywords | Most_Representative_Document | |
|---|---|---|---|
| Topic_Number | |||
| 0 | 0.1352 | story, ghost, love, end, scary, play, short, world, time, start, creepy, work, suspense, bite, funny, moment, hollywood, atmosphere, genre, canst | zombie plenty thistle cheerleader splendid gods gift debut story peter end |
| 1 | 0.1492 | house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character | soon plot tranquil set family soon silence shatter event unit fall apart normal usual nowadays marvel fantasy disney release week usual helen murren samuel jackson jump consequence violence blood sexiness zaniness rest break ordinary case masterful let freely admit man power legs breasts hair hike skirt sex excited cold fish cut |
| 2 | 0.1617 | scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty | unlike reviews ism cheap shot ju-on fan supernatural owe decide wholeheartedly agree uncomfortable watch experience powerful ring dark water exorcista scary scary stuff |
| 3 | 0.1515 | watch, original, fan, ism, dont, youre, remake, minute, flick, ive, dead, people, enjoy, bad, movies, classic, time, wasnt, love, expect | soon plot tranquil set family soon silence shatter event unit fall apart normal usual nowadays marvel fantasy disney release week usual helen murren samuel jackson jump consequence violence blood sexiness zaniness rest break ordinary case masterful let freely admit man power legs breasts hair hike skirt sex excited cold fish cut |
| 4 | 0.1709 | scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow | different zombies army undead create industrial mishap result overwhelm chaos danger address morality root cause personally love pretty woman enjoy leg beautiful hair girl hat |
| 5 | 0.1399 | performance, actor, creepy, story, room, years, release, time, year, night, young, genre, director, play, stand, treat, fan, set, title, cinematography | fantastic performance start james mcevoy reason watch personality display distinct interest watch anya breath-taking witch fine job took night shyamalan reassert director follow string poor absolutely wait sequel conclusion unbreakable series arrive january |
| 6 | 0.1538 | character, people, dont, sound, live, thriller, kill, love, killer, play, end, feel, situation, place, time, world, school, problem, young, high | ive read review ism die hard thriller fan think slow- bad ism build ending actress play eden think cast better ill worth watch thrillers |
| 7 | 0.1423 | time, feel, leave, doesnt, character, story, dont, watch, sense, start, happen, end, didnt, friend, real, understand, scary, people, characters, hard | point mince words brad anderson session ive time intelligent well-written completely unpredictable look didnt notice view edit photography work soundtrack downright creepy recently manage lie awake night dario argentous opera tobe hoopers texas chain saw massacre list include honestly excuse folks doesnt |
In [61]:
lda_mallet_topic_dist = get_topic_distribution(lda_mallet_dom_topics_final, lda_mallet_most_representative_docs, n_topics=20)
lda_mallet_topic_dist.style.applymap(color_green).applymap(make_bold)
Out[61]:
| Documents_per_Topic | Percent_of_Total_Corpus | Topic_Keywords | |
|---|---|---|---|
| Topic_Number | |||
| 0 | 120 | 0.12 | story, ghost, love, end, scary, play, short, world, time, start, creepy, work, suspense, bite, funny, moment, hollywood, atmosphere, genre, canst |
| 1 | 110 | 0.11 | house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character |
| 2 | 133 | 0.133 | scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty |
| 3 | 201 | 0.201 | watch, original, fan, ism, dont, youre, remake, minute, flick, ive, dead, people, enjoy, bad, movies, classic, time, wasnt, love, expect |
| 4 | 84 | 0.084 | scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow |
| 5 | 96 | 0.096 | performance, actor, creepy, story, room, years, release, time, year, night, young, genre, director, play, stand, treat, fan, set, title, cinematography |
| 6 | 110 | 0.11 | character, people, dont, sound, live, thriller, kill, love, killer, play, end, feel, situation, place, time, world, school, problem, young, high |
| 7 | 146 | 0.146 | time, feel, leave, doesnt, character, story, dont, watch, sense, start, happen, end, didnt, friend, real, understand, scary, people, characters, hard |
In [68]:
#NOTE: .malletmodel2ldamodel() is buggy!
#Cf. https://github.com/RaRe-Technologies/gensim/issues/1203
#Cf. https://github.com/RaRe-Technologies/gensim/issues/2069
#For workaround, Cf. http://jeriwieringa.com/2018/07/17/pyLDAviz-and-Mallet/#comment-4018495276
#mallet_2_lda_mod = gensim.models.wrappers.ldamallet.malletmodel2ldamodel(lda_mallet)
#prepared = pyLDAvis.gensim.prepare(mallet_2_lda_mod, corpus, dictionary)
#display(pyLDAvis.display(prepared))
LDA Mallet: Second Pass with second highest C_V Coherence Optimized number of topics¶
In [62]:
lda_mallet_2 = LdaMallet(mallet_path, corpus=corpus, num_topics=14, id2word=dictionary)
In [64]:
lda_mallet_df_2 = get_topics(lda_mallet_2, 14)
#print(lda_mallet_df_2)
lda_mallet_df_2
Out[64]:
| Topic # 01 | Topic # 02 | Topic # 03 | Topic # 04 | Topic # 05 | Topic # 06 | Topic # 07 | Topic # 08 | Topic # 09 | Topic # 10 | Topic # 11 | Topic # 12 | Topic # 13 | Topic # 14 | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | style | original | dont | scare | work | story | scene | people | story | sound | house | people | zombie | family |
| 1 | vampire | watch | character | scary | nature | end | story | story | ghost | time | didnt | feel | people | youre |
| 2 | scene | fan | room | ive | woman | play | dark | watch | sinister | love | story | friend | shoot | work |
| 3 | feel | remake | time | jump | wrong | short | ring | scene | love | character | feel | character | dead | twist |
| 4 | cinematography | classic | thriller | watch | fall | fun | real | turn | director | ism | plot | leave | days | expect |
| 5 | brilliant | flick | ive | haunt | describe | time | watch | character | scary | dont | girl | real | ride | start |
| 6 | excellent | dead | story | real | director | creepy | sense | viewer | home | live | guy | japanese | attack | add |
| 7 | ism | treat | past | creepy | mind | director | plot | reason | fan | monster | happen | life | scene | end |
| 8 | music | gore | play | time | fear | bite | water | blood | cast | hear | kid | world | character | dont |
| 9 | tale | year | work | doesnt | include | start | visual | truth | atmosphere | pretty | night | act | tension | characters |
| 10 | line | evil | doesnt | extremely | case | surprise | time | plot | family | face | review | shock | villain | time |
| 11 | set | release | 1408 | vampire | pain | funny | david | feel | war | walk | start | time | completely | act |
| 12 | bad | trick | writer | conjuring | child | pretty | experience | game | child | start | break | sarah | point | minute |
| 13 | genre | enjoy | things | performance | death | suspense | point | idea | supernatural | leave | leave | juno | action | doesnt |
| 14 | place | fans | write | genre | characters | bring | face | camera | element | follow | enjoy | high | town | light |
| 15 | beautiful | performance | twist | james | emotion | minute | resolution | body | screen | world | end | kill | woman | strong |
| 16 | atmosphere | genre | ll | theyre | love | night | sound | review | orphanage | kill | sense | happen | virus | pace |
| 17 | girl | sequel | john | feel | psychological | gore | realize | raw | house | loud | bad | year | infect | build |
| 18 | absolutely | teen | read | music | hard | clich | weird | isnt | simple | human | money | event | person | rest |
| 19 | kind | fun | director | scares | emotional | fan | mike | work | young | eat | run | young | begin | genre |
In [115]:
ldamallet_hellinger_topic_distances_2 = get_most_similar_topics(lda_mallet_2, lda_mallet_df_2, num_topics=14, mallet_flag=True)
pretty_print(ldamallet_hellinger_topic_distances_2)
Hellinger Distances between topics are: ______________________________________________________________________ (0.9640064905425804, 'Topic # 02', 'Topic # 07') (0.9630872911656057, 'Topic # 05', 'Topic # 11') (0.959551339144133, 'Topic # 04', 'Topic # 05') (0.9583074176732344, 'Topic # 09', 'Topic # 13') (0.9572472126294248, 'Topic # 02', 'Topic # 11') (0.9572410852906693, 'Topic # 06', 'Topic # 13') (0.9564785062656007, 'Topic # 09', 'Topic # 11') (0.9515582230378915, 'Topic # 04', 'Topic # 10') (0.9512168721624514, 'Topic # 04', 'Topic # 13') (0.9502617007912305, 'Topic # 09', 'Topic # 10') (0.9499244066667502, 'Topic # 02', 'Topic # 05') (0.9468286136008315, 'Topic # 02', 'Topic # 13') (0.9430261325783785, 'Topic # 05', 'Topic # 06') (0.9425823285725585, 'Topic # 06', 'Topic # 12') (0.9420040275533847, 'Topic # 01', 'Topic # 05') (0.9417975252917109, 'Topic # 02', 'Topic # 03') (0.9402472926059382, 'Topic # 02', 'Topic # 08') (0.9402072660218292, 'Topic # 13', 'Topic # 14') (0.9397780668554646, 'Topic # 02', 'Topic # 10') (0.9380780343337605, 'Topic # 02', 'Topic # 12') (0.9380136294785817, 'Topic # 02', 'Topic # 06') (0.936734608528866, 'Topic # 01', 'Topic # 03') (0.9366856750100395, 'Topic # 01', 'Topic # 02') (0.9366577936877349, 'Topic # 05', 'Topic # 09') (0.9364644951443643, 'Topic # 01', 'Topic # 13') (0.9363631057812704, 'Topic # 08', 'Topic # 10') (0.9362873858656198, 'Topic # 03', 'Topic # 05') (0.9360853350982958, 'Topic # 04', 'Topic # 11') (0.9357046974338342, 'Topic # 07', 'Topic # 11') (0.9352819159927208, 'Topic # 05', 'Topic # 07') (0.934566993652253, 'Topic # 02', 'Topic # 14') (0.9343355961517718, 'Topic # 01', 'Topic # 06') (0.9342267700044657, 'Topic # 11', 'Topic # 13') (0.9339365932274444, 'Topic # 01', 'Topic # 10') (0.9326864835629602, 'Topic # 05', 'Topic # 13') (0.9324200074954956, 'Topic # 06', 'Topic # 07') (0.9320295173396599, 'Topic # 10', 'Topic # 13') (0.9310044895645258, 'Topic # 04', 'Topic # 12') (0.9310004677761589, 'Topic # 06', 'Topic # 10') (0.9307135461378772, 'Topic # 08', 'Topic # 09') (0.9305707470324849, 'Topic # 04', 'Topic # 08') (0.9299139984274493, 'Topic # 06', 'Topic # 08') (0.928261964297447, 'Topic # 01', 'Topic # 09') (0.9280944444529539, 'Topic # 09', 'Topic # 12') (0.9277725014527576, 'Topic # 07', 'Topic # 10') (0.9276461452648772, 'Topic # 07', 'Topic # 13') (0.9269518510732289, 'Topic # 08', 'Topic # 14') (0.9268323096340466, 'Topic # 01', 'Topic # 07') (0.9264987433284755, 'Topic # 02', 'Topic # 09') (0.9262790258347953, 'Topic # 07', 'Topic # 09') (0.9255696046789911, 'Topic # 03', 'Topic # 13') (0.9253181565349187, 'Topic # 01', 'Topic # 14') (0.9250726011887772, 'Topic # 01', 'Topic # 11') (0.9248987448461002, 'Topic # 04', 'Topic # 09') (0.924804376326733, 'Topic # 10', 'Topic # 11') (0.9241945250457422, 'Topic # 05', 'Topic # 14') (0.9236020907564638, 'Topic # 05', 'Topic # 08') (0.923549657003945, 'Topic # 06', 'Topic # 11') (0.9227998720545468, 'Topic # 04', 'Topic # 07') (0.9225210323813859, 'Topic # 05', 'Topic # 10') (0.9210296589008242, 'Topic # 02', 'Topic # 04') (0.9205754795844071, 'Topic # 07', 'Topic # 14') (0.918901528198737, 'Topic # 05', 'Topic # 12') (0.9188051769175941, 'Topic # 01', 'Topic # 08') (0.9182552888186247, 'Topic # 09', 'Topic # 14') (0.9178322793105529, 'Topic # 08', 'Topic # 11') (0.9173822837777433, 'Topic # 04', 'Topic # 14') (0.9166991032071017, 'Topic # 11', 'Topic # 14') (0.9160713848527243, 'Topic # 03', 'Topic # 10') (0.9156962901359114, 'Topic # 06', 'Topic # 09') (0.9154662018446263, 'Topic # 10', 'Topic # 14') (0.9150848628709455, 'Topic # 03', 'Topic # 07') (0.9139783821894085, 'Topic # 03', 'Topic # 08') (0.9139618472217662, 'Topic # 12', 'Topic # 14') (0.9124574201528883, 'Topic # 01', 'Topic # 04') (0.9118831477143167, 'Topic # 03', 'Topic # 06') (0.910369561902317, 'Topic # 01', 'Topic # 12') (0.9090191200018434, 'Topic # 04', 'Topic # 06') (0.9085294755991589, 'Topic # 07', 'Topic # 08') (0.9076584663540122, 'Topic # 08', 'Topic # 13') (0.9068081504990716, 'Topic # 06', 'Topic # 14') (0.9060864158201576, 'Topic # 03', 'Topic # 09') (0.9053152568445154, 'Topic # 07', 'Topic # 12') (0.904457235584954, 'Topic # 03', 'Topic # 12') (0.9020310763313824, 'Topic # 12', 'Topic # 13') (0.8977620593410172, 'Topic # 11', 'Topic # 12') (0.8957287994479677, 'Topic # 10', 'Topic # 12') (0.8928704047386233, 'Topic # 03', 'Topic # 14') (0.8902855908351145, 'Topic # 03', 'Topic # 11') (0.8834213857263032, 'Topic # 03', 'Topic # 04') (0.8798037125279303, 'Topic # 08', 'Topic # 12') ______________________________________________________________________
In [66]:
lda_mallet_dom_topics_df_2, lda_mallet_dom_topics_final_2 = get_dominant_topics(lda_mallet_2, corpus, all_reviews_joined)
In [67]:
#lda_mallet_dom_topics_final_2.style.applymap(color_green).applymap(make_bold)
lda_mallet_dom_topics_final_first30_2 = lda_mallet_dom_topics_final_2.head(30)
lda_mallet_dom_topics_final_first30_2.style.applymap(color_green).applymap(make_bold)
Out[67]:
| Dominant_Topic | Probability_Contribution | Topic_Keywords | Original_Text | |
|---|---|---|---|---|
| Document_Number | ||||
| 0 | 6 | 0.449 | scene, story, dark, ring, real, watch, sense, plot, water, visual, time, david, experience, point, face, resolution, sound, realize, weird, mike | coherent plot tell non-straightforward fashion open interpretations leave require audience fill-in open end possibly filling-in require viewings considerable think sense watch wailing time impression kind watch ponder read watch attempt people interpret explain different conclusion coherent underlie plot matt shift piece puzzle attempt recreate coherent narrative piece fit incompetence underlie story possibly intentionally possibly fundamentally inconsistent optical illusion escher drawing appear describe physical object fact dont physical sense accordingly enjoy boil content technically well-made cinematography acting costumes plot sense tell deceptive lure think plot sense matt sufficient thought leave lingering irritate feel dissatisfaction confusion maybe precisely point tell story purpose instill audience feel confusion face sequence event sense life times |
| 1 | 6 | 0.1745 | scene, story, dark, ring, real, watch, sense, plot, water, visual, time, david, experience, point, face, resolution, sound, realize, weird, mike | spoiler review extraordinary hour technically-perfect humanistic fine writer directors business hauteur saw devil cipher close analog suggest david lynches 2001 mulholland drive technically perfect humanistic suspense opus captivate length walk theatre shake head wonder saw film-makers understand secret story tell present story viewer feel share experience protagonist tell story viewer going provide explanation wailing means re-quote writer director numerous interview enigmatically begin wonder nature god -- recommended levels entertainment puzzled clinic engage wont let |
| 2 | 6 | 0.2982 | scene, story, dark, ring, real, watch, sense, plot, water, visual, time, david, experience, point, face, resolution, sound, realize, weird, mike | wailing open quote bible easy forget fact watch certain point clear purpose quote served warning religious person scare wailing work religious parables overcome lengthy run time tonal imbalanced deliver funny truly terrify mysterious strange small village south korean village plague sickness police think wild mushroom blame police officer jong-goo think stranger meet woman information mysterious man slowly begin fall rabbit hole consume life daughter start sign sickness personal figure trust avoid director hong-jin nap struggle tonal balance time pitch perfect time place tone struggle stay consistent primarily funny brief burst depravity violence comedy work laugh numb times horrific imagery sudden effective balance lose act comedy slapstick fit rest character over-the-top act exaggerate different introduced middle section moment unintentional humor example man strike lighten caller id shaman phone shaman fortunately balance act smartly switch act intense terrify moment comedy offset terrify act start unfold audience gain new appreciation rest start reinterpret certain scenes problem scene sense ended feel natural fit story progressed require end sense hour minutes middle portion resolved shaman screen time perform ritual end important zombie scene feel awkward didnt fit rest want zombie grow popularity scene bring numb random character serve purpose rest despite scene previously mention overacting act generally speak fantastic father task solve happen daughter convey terror hatred build intense persona carry priest train advice father equally effective bring concern innocent quality terror ultimately happen young daughter sick shines channeling inner linda blair emphatically deliver horrible dirty line child performance truly terrify watch hatred eye slowly end religious overtone clearer locusts attack individual white black suggest characters true nature scene truly shines slowly unveil real nature certain characters minute change perspective things minute switch determine trust reveal realize detail setup reveal twist lead ultimately putt faith wrong people wailing 2016 directed hong-jin na screenplay hong-jin na starring woo-hee jeong-min hwang so-yeon range won krak run time hour minute |
| 3 | 0 | 0.1639 | style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind | hell ride minute stop look time leave relieved left thats engage interest story watch read review smorgasbord genre positive review doesnt feel means yes storytelling fluid write multiple genre aspect didnt feel place works work feel redundant point genres similar witch necessarily term storyline pace severity terror psychological wailing deliver somewhat complex storyline cinematography breathtaking jun kunimura ridiculous oh-so-good actor point ism point mention asian technically guess lol dont think necessary pigeonhole |
| 4 | 10 | 0.1271 | house, didnt, story, feel, plot, girl, guy, happen, kid, night, review, start, break, leave, enjoy, end, sense, bad, money, run | want start review completely jump scares play integral movies rely support story leave displeased wailing special jump scare instead viewers story rich mystery character struggles dark atmosphere design amazingly craft scene blood run cold addition carry subtext leave viewer question evidence interpret feel director time research real experience possible wailing bite bore slow burner construct modern running time th 36min patient man pay end genre dead look cash grab mainstream wide release hide gems |
| 5 | 12 | 0.3841 | zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin | ism biased set city live work oxford street piccadilly circus pass morning usually teem crowd people completely send shiver spine usually watch locate nondescript midwestern village easy detach event unfold screen occur place home entirely new sense reality previously unaccustomed judge days entirely merit easy arrive conclusion fantastic achievements coming-of-age sort director danny boyle canst mtv-inspired vanity beach self-consciously trendy posture trainspotting appeal shame initially expect days similar treatment thankfully fear prove unfounded discard straight open sequence effortless fearsome rest joy terrify joy joy nonetheless true minimalism effective overblown bravado definitely true scene complete silence entire metropolis grainy picture serve add documentary-style quality situation real bear definitely wise choice digital video occasionally meet people think days wasnt scary gory people tell 2001 boring memento confusing ignore didnt understand purpose confuse change pace feel distract personally regard important reviewer point succinct point scarier end world world repopulate maniacs think real days lies 28 days romero zombie flick yore ultimately allegory day live age constantly confront violence medium ape right start violence beget violence humanity face uncertain future applaud danny boilers bravery days undoubtedly commercial risk majority cinema-going public prefer escapism word caution remember rage human-made disease allegory masterpiece time days misunderstand considerable people doubt history classic perfectly sum confuse era live didnt advisable days chance haunt experience look right angle danny boyle year leave hope hell continue |
| 6 | 12 | 0.3705 | zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin | virus rage virus infect person mad extreme rage hungry blood day outbreak london cause entire britain dead evacuate leave blood-thirsty infect population handful solitary normal persons civilization halt society destroy limit survivor fight existence frequent attack vicious victims sounds familiar 28 days classic horde zombie biohazard movies touch art danny boyle able bring focus extensive special-effects-controlled gory action sequence infect normals heavy background music theres tinge sadness emptiness helplessness consider london scene background music theres electrify action gore describe picture life condition talk mistake loophole couldnt weaken tight-gripping storyline achievement viewer stay neutral place virus talk metaphor rage social disease flick person harsh critic zombie watch stars mention billian murphy awesome |
| 7 | 2 | 0.1472 | dont, character, room, time, thriller, ive, story, past, play, work, doesnt, 1408, writer, things, write, twist, ll, john, read, director | ism amaze negative review think succeed level artistic atmospheric pace sympathetic characters fantastic climax music nicely eerie open scene london street worth price admission ism stubborn viewer normally benefit early critical buzz manner excuse time ism completely impressed incidentally think interest day inspire knockoffs knockoffs 28 days apparently owe john wyndham classic disaster novel day triffids thats things highly recommended |
| 8 | 12 | 0.3971 | zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin | 2003 state-side release danny boilers 28 days advertise chockful scare-fest didnt day ago gotta feel somewhat embellishment promoters environmental terrorist attack lab contain diseased chimp infect rage virus unwittingly let loose plague lay waste england rest society days title cut abandon london coma-tized bicycle courier jim cillian murphy effective wake stasis hospital search london sign life rescue rage zombie couple survivalists lovely naomi harris follow place place alive meet man daughter brendan gleeson terrific megan burns refuge london-town record message paradise salvation track frank man shortwave radio feel meditation happen people reduce low elements friend tell movies run zombie inspire zombie remake dawn dead dawn pretty full-throttle action hybrid start finish play emphasis creature lucky survivors disturb lesson nature survival interest disturb flick sell wrong |
| 9 | 12 | 0.4656 | zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin | key sci-fi genre alive cinemas material technique filmmakers present competent average creative havenet havenet style form george romero prime era film-making bring forth memorable trilogy time genre consider romero list director included important source independent 1968 inventive result masterpiece lacklusters director danny boyle author alex garland cook yarn similar romero wouldnt satisfying theyve essential success days later- idea practice years turn fresh audience feel repelled excited terrified nauseous enthralled wont leave feel weave complete hack work boyle team set freshen zombie string precise term zombie movie- hear living-dead utter hear infected new word people rage infect britain monkey virus let loose island begin infectious spread cut man jim lie hospital bed wander abandon street view tear fragment society boyle implement atmosphere heavily action chase scene indication dedication details jim soon survivors include selena naomie harris father daughter brendan gleeson megan burns hear salvation radio decide brave military outpost thats true salvation outside military typos boyle producer andrew macdonald spellbind thats proper terminology adaptation trainspotting craft par arguably toppings story characters captivate care plight jim companions photography anthony dod mantle striking digital photography blair witch use non-professional camera equipment add proper shade needed shoot composition different edit chris gill quick expect attack scenes fast infect throw blood scream on-looker new infect transform seconds result follow mark tildesley production designs john murphys music evoke haunting evocative mood mundane scenes acting consider actor well-known believable script ism days cup tea science fiction fan immediately hear preview tv add dismiss phooey rubble borrow video-store point view similarity romero military scene remind day dead chain zombie practical reasons supermarket scene unneeded consider satirical reverence dawn dead understand boyle isnt 100 original point genres history credible job different tone different set country different envelop view scene structures overall days construct execute sci-fi youve seen sci-fi youve combined sense modern audience fascinate |
| 10 | 0 | 0.3686 | style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind | let right vampire seen smarter scary nuanced doesnt feel thriller feel literature detail bizarre misadventure pair pre-teen star cross lovers androgynous vampired phenomenal regard detail young oskar kare hedebrant life spot stuck incredibly painful period post-childhood pre-adolescence oskar aware girls idea contend small age brutalize boy result terribly collect news clip violent crime let rage day strange young girl eli klina leandersson appear playground fast friend begin look oskar eli innocently spend night occasion privy oskar doesnt happens elias caretaker serial killer brutal order desanguinating victim bucket soon oskar realize new friend bite tragedy shock violence eli leave fend desperately stave urge drink fresh blood form delicate new bond oskar remake let right dont wait hope capture magic original gorgeously shot impeccably written fully realize unmatched respect vampire lore acting producer amaze young actors odd recapture brilliant melancholy chemistry astronomical let right think traditional opt endless ultra violence cut favor kiddie friendly rating director tomas alfredson steer line right middle violence brutal horrific dwell leave question kidney display stillness mimic oskar stage stuck moving hope growth begin changes apparent stillness oskar elik oskar grow change man eli stick burdensome fate quiet understate ending haunting whimsical forth times people understanding kind people beg dont miss time silence lambs real chance home oscar gold deserve a+ |
| 11 | 0 | 0.2909 | style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind | let right heart sweet coming-of-age story unique different defy categorizations swedish adapt john aside lindqvist bestselling book director tomas alfredson dare mix pleasure pain horrify tender let right storyline reveal secret early leave surprise necessarily rare review plot better 12-year-olds oskar kare hedebrant eli klina leandersson meet snowy afternoon jungle gym courtyard oskar house complex outside stockholm young tender attraction apparent right start think relationship headed deep dark secret discover reveal audience repulse curiously fascinate time similar fashion yellow crime scene tape bring close warn support cast completely beholden narrative revolve adorable young couple performance rival ive actor age innocence vulnerability hedebrant oskar tour-de-force admirably carry shoulders leandersson match scene scene line line result literally chills production value stellary technical aspect -- lighting original music johan soderqvist hoyle van hoytema cinematography -- combine perfect synchronization produce hitchcockian tale bring love light dark drama imaginable let right overwhelm choice best narrative feature north american premiere 2008 tribes festival truly well-deserved honor tomas alfredson craft brilliant work art leave shake head wonder |
| 12 | 0 | 0.1836 | style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind | tomas alfredson let right original dark twist gory fantasy special hard classify merely exercise style brilliant piece amoral storytellings characters action defy logic common sense dont wanna spoil moment ll mean revenge moment story happens remind fantasy tale oskar kåre hedebrant year-old bully boy befriend develop innocent crush new neighbor eli klina leandersson happen vampired twist tale revenge pubescent love visual flair swim pool scene classic creative direct impressive performance young pair protagonists hollywood course didnt waste time announce upcoming remake lazy read subtitles likely remake turn pg-13 order money fill moral value prudish parent let kid watch dont vampired year ago ignore future bomb enjoy original youre treaty |
| 13 | 0 | 0.4459 | style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind | people review idb focus sweet friendship year old human vampire leads huge element sweet story childhood friendship godfather story father concern worry kids true descriptions miss point think seen truth let right dark measure brilliant story tell week ago disturb echo stay refuse enjoy prior knowledge impossible review properly disclose plot elements warn spoilers ahead comment said character hokan key story end lead believe oskar step place fact reveal final scene dark nightmarish core blossom friendship witness prior offer redemption oskar path relationship eli takes håkan butcher boy old oskar food elik story storytelling high order dark whisper hint suggest questions allude play mind answered hokan pick victim presumably elias preference elias interest oskar elias true nature time briefly glimpse true physical age fact late middle age roughly age håkan gift oskar moments brutal slaughter child tormentors entranced glamour oskar happy happy happily sunset oskar presumably deaden soul destroy husk person hokan butcher fellow human provide eli food puzzle people think sweet fact conclusion utterly chilling oskar happily swap commonplace misery childhood bully fate truly hell earth happily smile shoot doesnt clue superlative heartily recommends tale succeed suggestion play imagination truly addition vampire cannon variation bram stokers brilliant original |
| 14 | 1 | 0.1126 | original, watch, fan, remake, classic, flick, dead, treat, gore, year, evil, release, trick, enjoy, fans, performance, genre, sequel, teen, fun | fond vampire genre artistic poetic profound explore nature evil world child pure evil pure goodness somewhat discernible achieve astound array contrast allow evil coexist thought-provoking leave audience yearn read novel short gems need remake accord dont need wait 2010 watch |
| 15 | 10 | 0.1489 | house, didnt, story, feel, plot, girl, guy, happen, kid, night, review, start, break, leave, enjoy, end, sense, bad, money, run | zombies zombies cheap outta original idea moves mandatory width train play beautiful girl sucker long-haired woman short skirts babe morality story ethic means played |
| 16 | 4 | 0.125 | work, nature, woman, wrong, fall, describe, director, mind, fear, include, case, pain, child, death, characters, emotion, love, psychological, hard, emotional | different zombies army undead create industrial mishap result overwhelm chaos danger address morality root cause personally love pretty woman enjoy leg beautiful hair girl hat |
| 17 | 4 | 0.1189 | work, nature, woman, wrong, fall, describe, director, mind, fear, include, case, pain, child, death, characters, emotion, love, psychological, hard, emotional | train susan treat mind eyes look capitalism vs humanity man vs empathy remember time enjoy foreign act excellent effeminate young man story original definitely eye delicious cheerleader |
| 18 | 9 | 0.1322 | sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat | dont youths fun action-packed dangerous elements innovative pretty girl importantly successful yup hollywood remake entry series redundant unnecessary inferior whitewash woman western taliban style ugly hollywood remakes paul fig jar jar abrams negotiate cut speak |
| 19 | 2 | 0.0859 | dont, character, room, time, thriller, ive, story, past, play, work, doesnt, 1408, writer, things, write, twist, ll, john, read, director | zombie plenty thistle cheerleader splendid gods gift debut story peter end |
| 20 | 3 | 0.4558 | scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares | ve guess director saw end inventive filmmaker work industry james wan brilliantly retro day deliver extremely stylistic visually strike stand tall classics virtually sex gore curse conjuring earn r-rating scare set 1971 conjuring focus marry paranormal researcher ed patrick wilson lorraine vera farmiga warren lecture college interest case theyre think retirement cue perron family parent roger iron livingstone carolyn lili taylor scare live live daughter claim evil rhode island home doesnt warrens discover perrons torment supernatural wanting short conjuring terrify ive seen trying erase torture porn crowd prove difficult director saw doubt finally look pg normally frown junkiest despite sex gore swearing james wants late slap r rate youre wonder frighten think mpa speak behalf day climax middle turn follow doesnt affect conjuring comic relief brilliantly place bring climax chance build fear startle wan understand psychology tension build suspense mere scene construction obviously note exorcist amityville inspiration derive real case file warren famous case date straying ironic style famous cabin woods surface story inventive assure work creative recent memory ive love james wan manage unoriginal haunted-house- possession story cases weave scares pacing sound design camera work describe precise james wan cinematographer john leonetti responsible look insidious fresh visual style unlike include wide shot movement cinematography absolutely stunning rely shaky cam realism leave unique feel separate visually narratively relate toone issue ive recent genre reveal demon insidious sinister example hook story fear unknown filmmaker audience wan definitely learn insidious conjuring apparition arendt reveal audience theyre far focus letting use imagination truly horrifying think filmmakers note mrs wan want complaint isnt wrong market screw trailer conjuring reveal scares avoid trailer command james wants twitter account hard miss tv spots wish fresh mind havenet trailer want stay marketing ultimately overall production value allow conjuring stand rot genre act impressive practical effect perfect classic feel wan time movies must-see summer rating 10let twitter thejoshl think conjuring |
| 21 | 3 | 0.3445 | scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares | dont summon devil dont priest lucky conjuring preview screen brightest 2013 totally cold trailer story plot turn scary seen conjuring nail-biting hiding-behind-hands youve disappoint paranormal activity insidious likely deliver area failed tell supposedly true story paranormal investigators aim rid family property suspect supernatural visitations disprove turn creaky floorboard slam doors tackle head-on leap faith require buy theme youre okay play pretty genre confines particular incident bring deal describe surround spirit malevolent hide public fairly amityville-like circumstances family new house discover basement sealed boarded-up doorway surprise follows decide look basement surprise james wan manage tire theme haunt possession revive convincingly strange kind truly tension terror enjoy sinister recently equally scary lose bite end conjuring tempo fairly satisfy conclusion turn comical middle minutes fire cylinder head finale intentional lure false sense security work beautifully youre type poo-poo genre general canst suddenly convert believer enjoy classic exorcista amityville poltergeist guarantee conjuring wont disappoint |
| 22 | 3 | 0.2398 | scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares | ism avid fan lately ive think isnt scare sinister skin appreciate james wants love saw insidious damn modern ghost story review state kinda lose momentum final act conjuring better scarier tense insidious ism gonna limb years classic rule making bad guy fully audience plus suspense tell throw water bottle screen sheer terror scare finally happened fake jump scares earn r rate blood sex profanity terror build scary masterpiece script acting direction style tone notch wants camera-work far movies choice set stroke genius feel authentic real story set present wouldnt effective scary style making costuming hair style throwback exorcist amityville extremely minor flaws length repetitiveness resolution happen fast dont care general flaws perfect havenet james wants conjuring ive time time scare badly dont miss amaze theater view experienced |
| 23 | 3 | 0.3782 | scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares | key conjuring freshness evidence ream fan argue internet site new table fan hungry time conjuring right splinter deal admire day age stand tall proud ream remakes sequel teen friendly slasher haunt multiplex frequency days free gore sex automatically alienate portion lustful member fan based serve appetising scare cauldron suspense deliver plenty particular table forget based true story tag kind irrelevant new technological age sell gimmick mean story true play bit regardless hoax charge embellishments buy premises commit scary story director james wan reward compliant story essentially base investigation early seventy paranormal specialist ed lorraine warren aid perron family victim dreadful supernatural event rhode island home wan build deftly let perron family live believe dream home start happen wan build slow instances create palpable sense dread camera work intelligent moment maximum impact yearn warrens involved kill boo-jump scares place care marry superbly mount tension naturally clich convention haunt house strange smells creaky doors ominous cellar supplement wants talented knack scare effective production designs mysterious bruise literal leg pull breath hold game hide seek bona fide pant soil moments conjuring lesson sustain unease finale unleashed script devoid cheese pointless filler refresh sub-genre suffer problems joseph bishara musical score absolute nerve shredder refresh accompaniment doesnt resort telegraph shriek tell afraid overwhelm scene john leonetti cinematography gothic textures house outside lakeside farmhouses strong lead cast vera farmiga patrick wilson lili taylor ron livingston trump met critical box office success conjuring justify reputation superb haunt house |
| 24 | 3 | 0.3111 | scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares | conjuring high class hard scare care character story compel feel interest time based true life events ed lorraine warren paranormal investigator set help family terrorize demon terrify case live hadnt share love place 70 agree completely director james wants point view impossible set present example teenage daughter haunt family picture demon diphones post post instagram basically demon turn victim human bad guys isnt perfect thought plot hole scare jump seat build tension let character start craziness set 70 feel 70 slow zoom style wouldnt expect modern believe james wan outdo ill far scary ive life thank james wan luck fast furious d recommend look guess feel audience feel watch exorcist dont look poltergeist time |
| 25 | 9 | 0.265 | sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat | pit human horrendous extra terrestrial end cheap imitation aliens series pitch black stand fine piece sci-fi excellent favorite aspect lighting beautifully employ different colors shade intensity light set mood lend unique feel different normal light generally subject vin diesel bring character life excellent manner skillfully avoid routine portrayal harden criminal riddick diesel character personal journey thankfully vin doesnt drop ball remainder cast exception talented gorgeous claudia black unknown turn marvelous performances animate diverse character unique quirk mannerisms pitch black perfect example resource excessive budget special effect adequate time mean sole focus high budget blockbusters use science fiction medium tell engaging provoke story tell mediocre story use flash science fiction |
| 26 | 9 | 0.36 | sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat | pitch black survival story survive hostile alien world hostile enemies task difficult near enemy survive group plot pitch black usual time different variations shine well-written characters group consist different people interest jacks boy secret fryd pilot hard time conscience johns bounty- hunter drug-habit imams holy man face fact god cruel riddick convict murderer learn value characters start live arendt paper genre start care characters reddick feel odd suppose bad guy line light pitch black think character familiar work casting rarely actor fit role radha mitchell suitable fryd cole hauser bring right cruelness sense responsibility johns impressive work vin diesel job riddick hand reddick creepy definitely dangerous deep character suppose survive pitch black friend close enemy closer |
| 27 | 9 | 0.3351 | sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat | admit fan serenity reddick chronicles bite predetermine anticipation flip ism vin geisel fan somewhat mono-dimensional range characters embodiment riddick fit perfectly story play pace encumber gratuitous special effect fit cgi appropriate glare technical defect story start walk breathe air slight concern difference atmosphere microbe common flaw gloss stories canst harp detail radha mitchell smoking hot ridiculous frivolous tough season interplanetary merchant marine ships captain character fry nail perfectly remind andrea start hungarian assassin gilda tigers lady appear drop check heartbeat digress cases trouble grant star pitch black |
| 28 | 9 | 0.4401 | sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat | doubt excite satisfy ive years plot print banal- ship crash desert planet suns survivor adjust landscape darkness fall monster appear pilot fryd moment cowardice descent atmosphere jettison passengers charge group enlist help convict murderer reddick lead darkness escape ship surgically enhance eye dark simple character complex three-dimensional conflict trait whoas whoas bathe performance uniformly superb radha mitchell fry steel leadership overcome fears prevent bloodshed cole hauser man reddick custody agendum idiosyncratic standards belong vin diesel reddick magnetic screen presence ive years face shadow doesnt deal draw attention draw attention speak moving diesel doesnt trivialise character easily heart gold reddick mean vicious man approach ship let glimpse tentative step care self care technically light effect excellent end spectrum overbright triple sunlight pitch darkness special effect riddick monsters point view add suspense sound effect monster fly ultrasound monster anatomically plausible suitably frightening editing tight jar time literally pad fades time re-orient open shoot end credit wit scene line dialogue single camera shoot important time understand ally caveat science solar system model impossible planets revolve suns vice versa doesnt affect human story havenet point |
| 29 | 12 | 0.0948 | zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin | open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain |
In [68]:
lda_mallet_most_representative_docs_2 = get_most_representative_docs(lda_mallet_dom_topics_final_2, n_topics=20)
lda_mallet_most_representative_docs_2.style.applymap(color_green).applymap(make_bold)
Out[68]:
| Probability_Contribution | Topic_Keywords | Most_Representative_Document | |
|---|---|---|---|
| Topic_Number | |||
| 0 | 0.11 | style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind | think true original break sweat certain points physical effect audience astonish moment scene transition literally breathtaking didnt believe director certain solely shock people suggested look lush style stun compositions favourite story final sickly grandson original character source hukkle want director interest time linger mind |
| 1 | 0.1054 | original, watch, fan, remake, classic, flick, dead, treat, gore, year, evil, release, trick, enjoy, fans, performance, genre, sequel, teen, fun | create extremely tense atmosphere moment starts plenty gore violence bombard eyes mention twist watch night light expect freak think typical instead completely intrigue story line utterly draw 100 minutes end feel scared feel offer satisfaction leave slightly disturb thoughtful state nightmares plague day-dreams day thought stray far thats powerful want cringingly gory violent watch texas chainsaw massacre hostel want nightmarishly scary watch grudge ring want clever disturb excellent story twist watch saw |
| 2 | 0.0859 | dont, character, room, time, thriller, ive, story, past, play, work, doesnt, 1408, writer, things, write, twist, ll, john, read, director | zombie plenty thistle cheerleader splendid gods gift debut story peter end |
| 3 | 0.1174 | scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares | unlike reviews ism cheap shot ju-on fan supernatural owe decide wholeheartedly agree uncomfortable watch experience powerful ring dark water exorcista scary scary stuff |
| 4 | 0.1067 | work, nature, woman, wrong, fall, describe, director, mind, fear, include, case, pain, child, death, characters, emotion, love, psychological, hard, emotional | soon plot tranquil set family soon silence shatter event unit fall apart normal usual nowadays marvel fantasy disney release week usual helen murren samuel jackson jump consequence violence blood sexiness zaniness rest break ordinary case masterful let freely admit man power legs breasts hair hike skirt sex excited cold fish cut |
| 5 | 0.0836 | story, end, play, short, fun, time, creepy, director, bite, start, surprise, funny, pretty, suspense, bring, minute, night, gore, clich, fan | plot summary describe story scary funny |
| 6 | 0.1194 | scene, story, dark, ring, real, watch, sense, plot, water, visual, time, david, experience, point, face, resolution, sound, realize, weird, mike | ive member idb year rarely time comment addition watch average 10-15 months split genre include lately thought ive disenfranchise proliferation shock hostel late saw captivity enter mist leave theater movies frank darabont adapt steven king novel short stories period bring human balance thats miss days unbelievable maybe ludicrous situation events character believable peel layer expose fear prejudice vulnerability foundation effective absolutely grip entire all-too-rare-these-days sense dread permeate scene leave emotionally exhaust end speak ending isnt talk ism opinion love split camp loved didn t conclusion entire storyline human condition wouldnt change things |
| 7 | 0.1102 | people, story, watch, scene, turn, character, viewer, reason, blood, truth, plot, feel, game, idea, camera, body, review, raw, isnt, work | usually comment read peoples comments finish watch rec feel comment heart rate let hell terrify think dont repeat synopsis people story doesnt original zombie claustrophobic set totally different brilliant let place apartment building limit space person shoot add claustrophobic atmosphere scare wont mile hollywood movies shocking ending flick seen literally believe hollywood watch learn dont need gallon blood body fly scare live hell |
| 8 | 0.1157 | story, ghost, sinister, love, director, scary, home, fan, cast, atmosphere, family, war, child, supernatural, element, screen, orphanage, house, simple, young | idea junks noise random jolts plenty fear deliver quietly compactly fuss suspenseful ive ring think better stomach knot prep chills rise wave calm water think bring robert dickman story near director uncanny skill place camera hold shot lead actress wonderful performance face elevator conclusion foreseeable--maybe end ghost story foreseeable--but nonetheless satisfying tale quietly disturb dread |
| 9 | 0.1078 | sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat | soon plot tranquil set family soon silence shatter event unit fall apart normal usual nowadays marvel fantasy disney release week usual helen murren samuel jackson jump consequence violence blood sexiness zaniness rest break ordinary case masterful let freely admit man power legs breasts hair hike skirt sex excited cold fish cut |
| 10 | 0.1017 | house, didnt, story, feel, plot, girl, guy, happen, kid, night, review, start, break, leave, enjoy, end, sense, bad, money, run | open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain |
| 11 | 0.1225 | people, feel, friend, character, leave, real, japanese, life, world, act, shock, time, sarah, juno, high, kill, happen, year, event, young | open scene guess familiar book said dont think possible satisfy adaption game thrones time need dwell novels create feel book tell story tell paper guess ill forgive filmmakers kid okay theyve manage portray loser relationship decently enough- consider run time time need development happen kid cut run time theyve change set 1958 bad choice changes theyre create scene arendt book theyre change problem execution constant jump scares cgi complete lack understand fact waiting scare scary throw constantly isnt terrifying alli didnt bored thought poor performance scary okay isnt adaptations |
| 12 | 0.0948 | zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin | open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain |
| 13 | 0.1074 | family, youre, work, twist, expect, start, add, end, dont, characters, time, act, minute, doesnt, light, strong, pace, build, rest, genre | scary want genuinely jump time watch twice mute sound laugh write trust scene challenge old tickers cardboard box hallway mind means use footage integrate overall extremely disturbing definite potential sequel story set new chapter id watch day time highly recommend jump-scare fans |
In [69]:
lda_mallet_topic_dist_2 = get_topic_distribution(lda_mallet_dom_topics_final_2, lda_mallet_most_representative_docs_2, n_topics=20)
lda_mallet_topic_dist_2.style.applymap(color_green).applymap(make_bold)
Out[69]:
| Documents_per_Topic | Percent_of_Total_Corpus | Topic_Keywords | |
|---|---|---|---|
| Topic_Number | |||
| 0 | 79 | 0.079 | style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind |
| 1 | 100 | 0.1 | original, watch, fan, remake, classic, flick, dead, treat, gore, year, evil, release, trick, enjoy, fans, performance, genre, sequel, teen, fun |
| 2 | 92 | 0.092 | dont, character, room, time, thriller, ive, story, past, play, work, doesnt, 1408, writer, things, write, twist, ll, john, read, director |
| 3 | 94 | 0.094 | scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares |
| 4 | 36 | 0.036 | work, nature, woman, wrong, fall, describe, director, mind, fear, include, case, pain, child, death, characters, emotion, love, psychological, hard, emotional |
| 5 | 84 | 0.084 | story, end, play, short, fun, time, creepy, director, bite, start, surprise, funny, pretty, suspense, bring, minute, night, gore, clich, fan |
| 6 | 63 | 0.063 | scene, story, dark, ring, real, watch, sense, plot, water, visual, time, david, experience, point, face, resolution, sound, realize, weird, mike |
| 7 | 65 | 0.065 | people, story, watch, scene, turn, character, viewer, reason, blood, truth, plot, feel, game, idea, camera, body, review, raw, isnt, work |
| 8 | 63 | 0.063 | story, ghost, sinister, love, director, scary, home, fan, cast, atmosphere, family, war, child, supernatural, element, screen, orphanage, house, simple, young |
| 9 | 60 | 0.06 | sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat |
| 10 | 78 | 0.078 | house, didnt, story, feel, plot, girl, guy, happen, kid, night, review, start, break, leave, enjoy, end, sense, bad, money, run |
| 11 | 57 | 0.057 | people, feel, friend, character, leave, real, japanese, life, world, act, shock, time, sarah, juno, high, kill, happen, year, event, young |
| 12 | 59 | 0.059 | zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin |
| 13 | 70 | 0.07 | family, youre, work, twist, expect, start, add, end, dont, characters, time, act, minute, doesnt, light, strong, pace, build, rest, genre |
HDP (Hierarchical Dirichlet Process)¶
Cf. http://proceedings.mlr.press/v15/wang11a/wang11a.pdf¶
In [70]:
hdp = models.HdpModel(corpus, id2word=dictionary, chunksize=1000, random_state=222)
In [100]:
#hdp_topics_matrix = hdp.show_topics(formatted=True, num_words=20)
In [101]:
#hdp_topics_matrix = np.array(hdp_topics_matrix)
#hdp_topics_matrix.shape
In [102]:
#hdp_topic_words = hdp_topics_matrix[:,1]
#for i in hdp_topic_words:
# print([reg_ex.sub('', w) for w in i.split() if len(reg_ex.sub('', w)) > 0])
# print('\n')
In [116]:
hdp_df = get_topics(hdp, 20)
#print(hdp_df)
hdp_df
Out[116]:
| Topic # 01 | Topic # 02 | Topic # 03 | Topic # 04 | Topic # 05 | Topic # 06 | Topic # 07 | Topic # 08 | Topic # 09 | Topic # 10 | Topic # 11 | Topic # 12 | Topic # 13 | Topic # 14 | Topic # 15 | Topic # 16 | Topic # 17 | Topic # 18 | Topic # 19 | Topic # 20 | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | room | story | scare | prof | park | water | character | salinsageonui | carmen | button | watch | remote-control | dont | boyle | tail | woods | lawn | weak-stomached | elves | rationalism |
| 1 | story | short | mysterious | laurier | metamorphosis | scene | strange | sewer | chainsaw | person | cinematically | benji | rowlands | progresses | resolution | starkly | twitch | dimension | roth | unearthly |
| 2 | 1408 | uncertainty | recovery | dont | talked | funeral | staples | idea | outcasts | plot | repopulate | shouts | department | hour | gozu | e-bay | unease | abuses | slicks | declining |
| 3 | character | role | gleeful | story | convict | sarsgaard | instance | plot-heavy | petty | live | burned | biblical | room | catwalk | inexplicable | wood | mentally | wee | hoodoo | meandering |
| 4 | hard-pressed | watch | feel | minami | ism | yoshimi | frightened | tab | se7en | essence | slapstick | two-story | police | rarely | walls | audience | vampire | centre | days | gate |
| 5 | play | fun | place | rides | exit | feel | time | angina | jolts | reveal | confessions | produce | someones | trash | calmly | george | ein | suspects | slam-bang | ignite |
| 6 | time | grotesque | favorite | think | attacks | diner | mix | visitors | girls | paste | fault | scandal | ride | post-birthday | titanic | tommy | wonder | issue | disturbance | starting |
| 7 | feel | pretty | styx | icon | abandonment | tight | novelized | faces | 911 | people | korea | archetype | mind-blowing | higher | fears | horrifying | horror-thrillers | simple-minded | et | malice |
| 8 | irreversible | theme | wonderous | le | imbalanced | adage | strong | london-town | hollow | su-mi | resist | jungle | expense | stringy | story | greatness | jeong-min | eager | red-eye | zack |
| 9 | work | time | look | mr | truth | chainsaws | isi | jesses | paula | jurassic | institute | shaman | 2010 | couple | fukikoshi | shower | estrange | toro | fanatical | lost--this |
| 10 | new | resistance | chance | texas | begun | misunderstand | sleep | eery | safe | advice | story | health | deep | jeonmal | carolyn | kiddie | porn | archetype | shouting | stinker |
| 11 | mist | handel | based | peasant | foggiest | silk | misshapen | response | require | murder | thailand | acceptable | plot | critic | lawyer | twitchers | freshment | choose | sanitize | heisman |
| 12 | similar | trade | deteriorate | marriage | cops | neighbours | human | decoration | nullify | check-out | bright | parallel | electric | trier | scene | exaggerate | leaver | plural | hellraiser | rich |
| 13 | cusack | purikitpanya | character | push | bolt | ing | faith | negotiable | stillness | hurricane | think | udo | payback | greatness | frantic | limp | professor | puzzle | keane | real |
| 14 | edition | behaviors | adage | melodramatics | impeccably | secretary | cock | produce | iranian | nights | finally | lineage | constantly | peace | theatrical | unsettling | spar | breathtaking | heighten | taper |
| 15 | ripped-to-shreds | victorian | clich | tightly-wired | limbless | outer | stylization | meshes | suppress | whilst | service | refuge | attempt | unnatural | hillside | disturb | provide | off-guard | atmosphere | ls054808375 |
| 16 | perversely | home | time | bewilder | people | dark | characters | so-so | vampire | states | wolf | loser | muse | actual | people | terrifying | obnoxious | help | spring | increase |
| 17 | noise | ameliorate | seek | titular | chase | afterthought | dont | grizzly | think | boast | festival | till | relaxed | connections | under-30 | consecutive | intend | gunshot | dreamy | perplex |
| 18 | progress | look | story | similarities | money-obsessed | self-sacrifice | lippi | hills | diverse | grainy | somethings | unexpectedly | decent | quarantine | oversights | foul-mouthed | depraved | plan | jumped | rapidly |
| 19 | bubblegum | dubbed | support | deceitfully | atv | reveal | phobia | spring | epilepticmoondancer | genitals | dispatch | palmas | 1408 | allow | third-wheel | basking | wes | 80s | love | exist |
In [117]:
hdp_hellinger_topic_distances = get_most_similar_topics(hdp, hdp_df, hdp_flag=True)
pretty_print(hdp_hellinger_topic_distances)
Hellinger Distances between topics are: ______________________________________________________________________ (0.18165902124584954, 'Topic # 06', 'Topic # 19') (0.18165902124584954, 'Topic # 01', 'Topic # 19') (0.1802775637731995, 'Topic # 17', 'Topic # 19') (0.1802775637731995, 'Topic # 06', 'Topic # 17') (0.1802775637731995, 'Topic # 01', 'Topic # 17') (0.17748239349298853, 'Topic # 19', 'Topic # 20') (0.17748239349298853, 'Topic # 06', 'Topic # 20') (0.17748239349298853, 'Topic # 02', 'Topic # 19') (0.17748239349298853, 'Topic # 02', 'Topic # 06') (0.17606816861659014, 'Topic # 18', 'Topic # 19') (0.17606816861659014, 'Topic # 17', 'Topic # 20') (0.17606816861659014, 'Topic # 06', 'Topic # 18') (0.17606816861659014, 'Topic # 02', 'Topic # 17') (0.17606816861659014, 'Topic # 01', 'Topic # 18') (0.17606816861659014, 'Topic # 01', 'Topic # 06') (0.17464249196572987, 'Topic # 17', 'Topic # 18') (0.17464249196572987, 'Topic # 16', 'Topic # 19') (0.17464249196572987, 'Topic # 14', 'Topic # 19') (0.17464249196572987, 'Topic # 13', 'Topic # 19') (0.17464249196572987, 'Topic # 06', 'Topic # 16') (0.17464249196572987, 'Topic # 06', 'Topic # 14') (0.17464249196572987, 'Topic # 01', 'Topic # 20') (0.17320508075688779, 'Topic # 16', 'Topic # 17') (0.17320508075688779, 'Topic # 14', 'Topic # 17') (0.17320508075688779, 'Topic # 13', 'Topic # 17') (0.17320508075688779, 'Topic # 12', 'Topic # 19') (0.17320508075688779, 'Topic # 11', 'Topic # 19') (0.17320508075688779, 'Topic # 10', 'Topic # 19') (0.17320508075688779, 'Topic # 06', 'Topic # 12') (0.17320508075688779, 'Topic # 06', 'Topic # 11') (0.17320508075688779, 'Topic # 02', 'Topic # 20') (0.17320508075688779, 'Topic # 01', 'Topic # 12') (0.17175564037317673, 'Topic # 18', 'Topic # 20') (0.17175564037317673, 'Topic # 12', 'Topic # 17') (0.17175564037317673, 'Topic # 11', 'Topic # 17') (0.17175564037317673, 'Topic # 10', 'Topic # 17') (0.17175564037317673, 'Topic # 06', 'Topic # 13') (0.17175564037317673, 'Topic # 06', 'Topic # 08') (0.17175564037317673, 'Topic # 02', 'Topic # 18') (0.17175564037317673, 'Topic # 01', 'Topic # 16') (0.17175564037317673, 'Topic # 01', 'Topic # 14') (0.17175564037317673, 'Topic # 01', 'Topic # 08') (0.17175564037317673, 'Topic # 01', 'Topic # 02') (0.17029386365926405, 'Topic # 16', 'Topic # 20') (0.17029386365926405, 'Topic # 15', 'Topic # 19') (0.17029386365926405, 'Topic # 14', 'Topic # 20') (0.17029386365926405, 'Topic # 13', 'Topic # 20') (0.17029386365926405, 'Topic # 08', 'Topic # 17') (0.17029386365926405, 'Topic # 06', 'Topic # 10') (0.17029386365926405, 'Topic # 02', 'Topic # 13') (0.16881943016134138, 'Topic # 16', 'Topic # 18') (0.16881943016134138, 'Topic # 15', 'Topic # 17') (0.16881943016134138, 'Topic # 14', 'Topic # 18') (0.16881943016134138, 'Topic # 13', 'Topic # 18') (0.16881943016134138, 'Topic # 12', 'Topic # 20') (0.16881943016134138, 'Topic # 11', 'Topic # 20') (0.16881943016134138, 'Topic # 10', 'Topic # 20') (0.16881943016134138, 'Topic # 08', 'Topic # 19') (0.16881943016134138, 'Topic # 02', 'Topic # 12') (0.16881943016134138, 'Topic # 02', 'Topic # 10') (0.16733200530681516, 'Topic # 13', 'Topic # 14') (0.16733200530681516, 'Topic # 11', 'Topic # 18') (0.16733200530681516, 'Topic # 10', 'Topic # 18') (0.16733200530681516, 'Topic # 08', 'Topic # 20') (0.16733200530681516, 'Topic # 06', 'Topic # 15') (0.16733200530681516, 'Topic # 02', 'Topic # 16') (0.16733200530681516, 'Topic # 02', 'Topic # 08') (0.16733200530681516, 'Topic # 01', 'Topic # 15') (0.16733200530681516, 'Topic # 01', 'Topic # 11') (0.16733200530681516, 'Topic # 01', 'Topic # 10') (0.16583123951777004, 'Topic # 15', 'Topic # 20') (0.16583123951777004, 'Topic # 12', 'Topic # 16') (0.16583123951777004, 'Topic # 12', 'Topic # 14') (0.16583123951777004, 'Topic # 12', 'Topic # 13') (0.16583123951777004, 'Topic # 11', 'Topic # 13') (0.16583123951777004, 'Topic # 10', 'Topic # 16') (0.16583123951777004, 'Topic # 10', 'Topic # 14') (0.16583123951777004, 'Topic # 08', 'Topic # 18') (0.16583123951777004, 'Topic # 03', 'Topic # 19') (0.16583123951777004, 'Topic # 01', 'Topic # 13') (0.16431676725154987, 'Topic # 15', 'Topic # 18') (0.16431676725154987, 'Topic # 13', 'Topic # 16') (0.16431676725154987, 'Topic # 12', 'Topic # 18') (0.16431676725154987, 'Topic # 11', 'Topic # 12') (0.16431676725154987, 'Topic # 10', 'Topic # 12') (0.16431676725154987, 'Topic # 10', 'Topic # 11') (0.16431676725154987, 'Topic # 09', 'Topic # 19') (0.16431676725154987, 'Topic # 08', 'Topic # 16') (0.16431676725154987, 'Topic # 08', 'Topic # 14') (0.16431676725154987, 'Topic # 08', 'Topic # 13') (0.16431676725154987, 'Topic # 06', 'Topic # 09') (0.16431676725154987, 'Topic # 03', 'Topic # 17') (0.16431676725154987, 'Topic # 02', 'Topic # 14') (0.1627882059609971, 'Topic # 13', 'Topic # 15') (0.1627882059609971, 'Topic # 11', 'Topic # 16') (0.1627882059609971, 'Topic # 11', 'Topic # 14') (0.1627882059609971, 'Topic # 08', 'Topic # 11') (0.1627882059609971, 'Topic # 08', 'Topic # 10') (0.1627882059609971, 'Topic # 04', 'Topic # 19') (0.1627882059609971, 'Topic # 04', 'Topic # 06') (0.1627882059609971, 'Topic # 02', 'Topic # 15') (0.1627882059609971, 'Topic # 02', 'Topic # 11') (0.16124515496597103, 'Topic # 14', 'Topic # 16') (0.16124515496597103, 'Topic # 07', 'Topic # 19') (0.16124515496597103, 'Topic # 06', 'Topic # 07') (0.16124515496597103, 'Topic # 04', 'Topic # 17') (0.16124515496597103, 'Topic # 03', 'Topic # 20') (0.16124515496597103, 'Topic # 01', 'Topic # 09') (0.15968719422671315, 'Topic # 15', 'Topic # 16') (0.15968719422671315, 'Topic # 14', 'Topic # 15') (0.15968719422671315, 'Topic # 10', 'Topic # 13') (0.15968719422671315, 'Topic # 09', 'Topic # 20') (0.15968719422671315, 'Topic # 09', 'Topic # 17') (0.15968719422671315, 'Topic # 08', 'Topic # 15') (0.15968719422671315, 'Topic # 08', 'Topic # 12') (0.15968719422671315, 'Topic # 07', 'Topic # 17') (0.15968719422671315, 'Topic # 03', 'Topic # 18') (0.15968719422671315, 'Topic # 03', 'Topic # 06') (0.15968719422671315, 'Topic # 02', 'Topic # 09') (0.158113883008419, 'Topic # 12', 'Topic # 15') (0.158113883008419, 'Topic # 11', 'Topic # 15') (0.158113883008419, 'Topic # 10', 'Topic # 15') (0.158113883008419, 'Topic # 09', 'Topic # 18') (0.158113883008419, 'Topic # 04', 'Topic # 20') (0.1565247584249853, 'Topic # 09', 'Topic # 16') (0.1565247584249853, 'Topic # 09', 'Topic # 14') (0.1565247584249853, 'Topic # 09', 'Topic # 13') (0.1565247584249853, 'Topic # 05', 'Topic # 19') (0.1565247584249853, 'Topic # 05', 'Topic # 06') (0.1565247584249853, 'Topic # 04', 'Topic # 18') (0.1565247584249853, 'Topic # 03', 'Topic # 12') (0.1565247584249853, 'Topic # 01', 'Topic # 05') (0.1565247584249853, 'Topic # 01', 'Topic # 04') (0.1549193338482967, 'Topic # 09', 'Topic # 12') (0.1549193338482967, 'Topic # 09', 'Topic # 10') (0.1549193338482967, 'Topic # 07', 'Topic # 18') (0.1549193338482967, 'Topic # 05', 'Topic # 17') (0.1549193338482967, 'Topic # 03', 'Topic # 16') (0.1549193338482967, 'Topic # 03', 'Topic # 14') (0.1549193338482967, 'Topic # 03', 'Topic # 13') (0.1549193338482967, 'Topic # 03', 'Topic # 08') (0.1549193338482967, 'Topic # 02', 'Topic # 04') (0.15329709716755896, 'Topic # 08', 'Topic # 09') (0.15329709716755896, 'Topic # 07', 'Topic # 20') (0.15329709716755896, 'Topic # 07', 'Topic # 16') (0.15329709716755896, 'Topic # 07', 'Topic # 14') (0.15329709716755896, 'Topic # 04', 'Topic # 12') (0.15329709716755896, 'Topic # 04', 'Topic # 10') (0.15329709716755896, 'Topic # 03', 'Topic # 11') (0.15329709716755896, 'Topic # 03', 'Topic # 10') (0.15329709716755896, 'Topic # 02', 'Topic # 07') (0.15329709716755896, 'Topic # 01', 'Topic # 03') (0.15165750888103105, 'Topic # 09', 'Topic # 15') (0.15165750888103105, 'Topic # 09', 'Topic # 11') (0.15165750888103105, 'Topic # 07', 'Topic # 12') (0.15165750888103105, 'Topic # 07', 'Topic # 11') (0.15165750888103105, 'Topic # 05', 'Topic # 20') (0.15165750888103105, 'Topic # 04', 'Topic # 16') (0.15165750888103105, 'Topic # 04', 'Topic # 14') (0.15165750888103105, 'Topic # 04', 'Topic # 13') (0.15165750888103105, 'Topic # 04', 'Topic # 08') (0.15165750888103105, 'Topic # 02', 'Topic # 05') (0.15165750888103105, 'Topic # 02', 'Topic # 03') (0.15165750888103105, 'Topic # 01', 'Topic # 07') (0.15000000000000002, 'Topic # 07', 'Topic # 08') (0.15000000000000002, 'Topic # 05', 'Topic # 18') (0.15000000000000002, 'Topic # 03', 'Topic # 15') (0.14832396974191328, 'Topic # 07', 'Topic # 15') (0.14832396974191328, 'Topic # 07', 'Topic # 10') (0.14832396974191328, 'Topic # 05', 'Topic # 16') (0.14832396974191328, 'Topic # 05', 'Topic # 14') (0.14832396974191328, 'Topic # 05', 'Topic # 13') (0.14662878298615184, 'Topic # 07', 'Topic # 13') (0.14662878298615184, 'Topic # 05', 'Topic # 12') (0.14662878298615184, 'Topic # 05', 'Topic # 11') (0.14662878298615184, 'Topic # 05', 'Topic # 10') (0.14662878298615184, 'Topic # 04', 'Topic # 15') (0.14662878298615184, 'Topic # 04', 'Topic # 11') (0.14662878298615184, 'Topic # 03', 'Topic # 09') (0.1449137674618944, 'Topic # 05', 'Topic # 08') (0.14317821063276356, 'Topic # 05', 'Topic # 15') (0.14142135623730953, 'Topic # 07', 'Topic # 09') (0.14142135623730953, 'Topic # 03', 'Topic # 04') (0.13964240043768944, 'Topic # 04', 'Topic # 09') (0.13784048752090225, 'Topic # 03', 'Topic # 05') (0.13601470508735444, 'Topic # 05', 'Topic # 09') (0.13601470508735444, 'Topic # 04', 'Topic # 07') (0.13601470508735444, 'Topic # 03', 'Topic # 07') (0.1341640786499874, 'Topic # 04', 'Topic # 05') (0.13228756555322954, 'Topic # 05', 'Topic # 07') ______________________________________________________________________
In [118]:
hdp_dom_topics_df, hdp_dom_topics_final = get_dominant_topics(hdp, corpus, all_reviews_joined)
In [119]:
#hdp_dom_topics_final.style.applymap(color_green).applymap(make_bold)
hdp_dom_topics_final_first30 = hdp_dom_topics_final.head(30)
hdp_dom_topics_final_first30.style.applymap(color_green).applymap(make_bold)
Out[119]:
| Dominant_Topic | Probability_Contribution | Topic_Keywords | Original_Text | |
|---|---|---|---|---|
| Document_Number | ||||
| 0 | 3 | 0.9912 | prof, laurier, dont, story, minami, rides, think, icon, le, mr, texas, peasant, marriage, push, melodramatics, tightly-wired, bewilder, titular, similarities, deceitfully | coherent plot tell non-straightforward fashion open interpretations leave require audience fill-in open end possibly filling-in require viewings considerable think sense watch wailing time impression kind watch ponder read watch attempt people interpret explain different conclusion coherent underlie plot matt shift piece puzzle attempt recreate coherent narrative piece fit incompetence underlie story possibly intentionally possibly fundamentally inconsistent optical illusion escher drawing appear describe physical object fact dont physical sense accordingly enjoy boil content technically well-made cinematography acting costumes plot sense tell deceptive lure think plot sense matt sufficient thought leave lingering irritate feel dissatisfaction confusion maybe precisely point tell story purpose instill audience feel confusion face sequence event sense life times |
| 1 | 48 | 0.6778 | bland, mani, onijala, multi-faceted, crawl, meaning-symbolism, keyboard, favourite, metaphor, sarsgaard, 1765, honk, elses, iowa, guy-friend, gladiator, treatment, 140, circumstances, notice | spoiler review extraordinary hour technically-perfect humanistic fine writer directors business hauteur saw devil cipher close analog suggest david lynches 2001 mulholland drive technically perfect humanistic suspense opus captivate length walk theatre shake head wonder saw film-makers understand secret story tell present story viewer feel share experience protagonist tell story viewer going provide explanation wailing means re-quote writer director numerous interview enigmatically begin wonder nature god -- recommended levels entertainment puzzled clinic engage wont let |
| 2 | 28 | 0.9969 | story, kurt, query, misspelling, recipient, work, rheda, sealed, steve, evict, act, flying, drink, daughter, nuts, gimmicks, scarcity, titillating, overlap, cartoony | wailing open quote bible easy forget fact watch certain point clear purpose quote served warning religious person scare wailing work religious parables overcome lengthy run time tonal imbalanced deliver funny truly terrify mysterious strange small village south korean village plague sickness police think wild mushroom blame police officer jong-goo think stranger meet woman information mysterious man slowly begin fall rabbit hole consume life daughter start sign sickness personal figure trust avoid director hong-jin nap struggle tonal balance time pitch perfect time place tone struggle stay consistent primarily funny brief burst depravity violence comedy work laugh numb times horrific imagery sudden effective balance lose act comedy slapstick fit rest character over-the-top act exaggerate different introduced middle section moment unintentional humor example man strike lighten caller id shaman phone shaman fortunately balance act smartly switch act intense terrify moment comedy offset terrify act start unfold audience gain new appreciation rest start reinterpret certain scenes problem scene sense ended feel natural fit story progressed require end sense hour minutes middle portion resolved shaman screen time perform ritual end important zombie scene feel awkward didnt fit rest want zombie grow popularity scene bring numb random character serve purpose rest despite scene previously mention overacting act generally speak fantastic father task solve happen daughter convey terror hatred build intense persona carry priest train advice father equally effective bring concern innocent quality terror ultimately happen young daughter sick shines channeling inner linda blair emphatically deliver horrible dirty line child performance truly terrify watch hatred eye slowly end religious overtone clearer locusts attack individual white black suggest characters true nature scene truly shines slowly unveil real nature certain characters minute change perspective things minute switch determine trust reveal realize detail setup reveal twist lead ultimately putt faith wrong people wailing 2016 directed hong-jin na screenplay hong-jin na starring woo-hee jeong-min hwang so-yeon range won krak run time hour minute |
| 3 | 5 | 0.9157 | water, scene, funeral, sarsgaard, yoshimi, feel, diner, tight, adage, chainsaws, misunderstand, silk, neighbours, ing, secretary, outer, dark, afterthought, self-sacrifice, reveal | hell ride minute stop look time leave relieved left thats engage interest story watch read review smorgasbord genre positive review doesnt feel means yes storytelling fluid write multiple genre aspect didnt feel place works work feel redundant point genres similar witch necessarily term storyline pace severity terror psychological wailing deliver somewhat complex storyline cinematography breathtaking jun kunimura ridiculous oh-so-good actor point ism point mention asian technically guess lol dont think necessary pigeonhole |
| 4 | 0 | 0.5896 | room, story, 1408, character, hard-pressed, play, time, feel, irreversible, work, new, mist, similar, cusack, edition, ripped-to-shreds, perversely, noise, progress, bubblegum | want start review completely jump scares play integral movies rely support story leave displeased wailing special jump scare instead viewers story rich mystery character struggles dark atmosphere design amazingly craft scene blood run cold addition carry subtext leave viewer question evidence interpret feel director time research real experience possible wailing bite bore slow burner construct modern running time th 36min patient man pay end genre dead look cash grab mainstream wide release hide gems |
| 5 | 96 | 0.8354 | head-on, carrier, formulaic, roughly, rendered, similar, celebrations, check-out, methodically, rosa, diva, lastly, veteran, narratively, chude, nagoya, asinine, ananda, critically, mimicking | ism biased set city live work oxford street piccadilly circus pass morning usually teem crowd people completely send shiver spine usually watch locate nondescript midwestern village easy detach event unfold screen occur place home entirely new sense reality previously unaccustomed judge days entirely merit easy arrive conclusion fantastic achievements coming-of-age sort director danny boyle canst mtv-inspired vanity beach self-consciously trendy posture trainspotting appeal shame initially expect days similar treatment thankfully fear prove unfounded discard straight open sequence effortless fearsome rest joy terrify joy joy nonetheless true minimalism effective overblown bravado definitely true scene complete silence entire metropolis grainy picture serve add documentary-style quality situation real bear definitely wise choice digital video occasionally meet people think days wasnt scary gory people tell 2001 boring memento confusing ignore didnt understand purpose confuse change pace feel distract personally regard important reviewer point succinct point scarier end world world repopulate maniacs think real days lies 28 days romero zombie flick yore ultimately allegory day live age constantly confront violence medium ape right start violence beget violence humanity face uncertain future applaud danny boilers bravery days undoubtedly commercial risk majority cinema-going public prefer escapism word caution remember rage human-made disease allegory masterpiece time days misunderstand considerable people doubt history classic perfectly sum confuse era live didnt advisable days chance haunt experience look right angle danny boyle year leave hope hell continue |
| 6 | 10 | 0.9913 | watch, cinematically, repopulate, burned, slapstick, confessions, fault, korea, resist, institute, story, thailand, bright, think, finally, service, wolf, festival, somethings, dispatch | virus rage virus infect person mad extreme rage hungry blood day outbreak london cause entire britain dead evacuate leave blood-thirsty infect population handful solitary normal persons civilization halt society destroy limit survivor fight existence frequent attack vicious victims sounds familiar 28 days classic horde zombie biohazard movies touch art danny boyle able bring focus extensive special-effects-controlled gory action sequence infect normals heavy background music theres tinge sadness emptiness helplessness consider london scene background music theres electrify action gore describe picture life condition talk mistake loophole couldnt weaken tight-gripping storyline achievement viewer stay neutral place virus talk metaphor rage social disease flick person harsh critic zombie watch stars mention billian murphy awesome |
| 7 | 3 | 0.984 | prof, laurier, dont, story, minami, rides, think, icon, le, mr, texas, peasant, marriage, push, melodramatics, tightly-wired, bewilder, titular, similarities, deceitfully | ism amaze negative review think succeed level artistic atmospheric pace sympathetic characters fantastic climax music nicely eerie open scene london street worth price admission ism stubborn viewer normally benefit early critical buzz manner excuse time ism completely impressed incidentally think interest day inspire knockoffs knockoffs 28 days apparently owe john wyndham classic disaster novel day triffids thats things highly recommended |
| 8 | 17 | 0.9922 | weak-stomached, dimension, abuses, wee, centre, suspects, issue, simple-minded, eager, toro, archetype, choose, plural, puzzle, breathtaking, off-guard, help, gunshot, plan, 80s | 2003 state-side release danny boilers 28 days advertise chockful scare-fest didnt day ago gotta feel somewhat embellishment promoters environmental terrorist attack lab contain diseased chimp infect rage virus unwittingly let loose plague lay waste england rest society days title cut abandon london coma-tized bicycle courier jim cillian murphy effective wake stasis hospital search london sign life rescue rage zombie couple survivalists lovely naomi harris follow place place alive meet man daughter brendan gleeson terrific megan burns refuge london-town record message paradise salvation track frank man shortwave radio feel meditation happen people reduce low elements friend tell movies run zombie inspire zombie remake dawn dead dawn pretty full-throttle action hybrid start finish play emphasis creature lucky survivors disturb lesson nature survival interest disturb flick sell wrong |
| 9 | 5 | 0.9968 | water, scene, funeral, sarsgaard, yoshimi, feel, diner, tight, adage, chainsaws, misunderstand, silk, neighbours, ing, secretary, outer, dark, afterthought, self-sacrifice, reveal | key sci-fi genre alive cinemas material technique filmmakers present competent average creative havenet havenet style form george romero prime era film-making bring forth memorable trilogy time genre consider romero list director included important source independent 1968 inventive result masterpiece lacklusters director danny boyle author alex garland cook yarn similar romero wouldnt satisfying theyve essential success days later- idea practice years turn fresh audience feel repelled excited terrified nauseous enthralled wont leave feel weave complete hack work boyle team set freshen zombie string precise term zombie movie- hear living-dead utter hear infected new word people rage infect britain monkey virus let loose island begin infectious spread cut man jim lie hospital bed wander abandon street view tear fragment society boyle implement atmosphere heavily action chase scene indication dedication details jim soon survivors include selena naomie harris father daughter brendan gleeson megan burns hear salvation radio decide brave military outpost thats true salvation outside military typos boyle producer andrew macdonald spellbind thats proper terminology adaptation trainspotting craft par arguably toppings story characters captivate care plight jim companions photography anthony dod mantle striking digital photography blair witch use non-professional camera equipment add proper shade needed shoot composition different edit chris gill quick expect attack scenes fast infect throw blood scream on-looker new infect transform seconds result follow mark tildesley production designs john murphys music evoke haunting evocative mood mundane scenes acting consider actor well-known believable script ism days cup tea science fiction fan immediately hear preview tv add dismiss phooey rubble borrow video-store point view similarity romero military scene remind day dead chain zombie practical reasons supermarket scene unneeded consider satirical reverence dawn dead understand boyle isnt 100 original point genres history credible job different tone different set country different envelop view scene structures overall days construct execute sci-fi youve seen sci-fi youve combined sense modern audience fascinate |
| 10 | 11 | 0.9355 | remote-control, benji, shouts, biblical, two-story, produce, scandal, archetype, jungle, shaman, health, acceptable, parallel, udo, lineage, refuge, loser, till, unexpectedly, palmas | let right vampire seen smarter scary nuanced doesnt feel thriller feel literature detail bizarre misadventure pair pre-teen star cross lovers androgynous vampired phenomenal regard detail young oskar kare hedebrant life spot stuck incredibly painful period post-childhood pre-adolescence oskar aware girls idea contend small age brutalize boy result terribly collect news clip violent crime let rage day strange young girl eli klina leandersson appear playground fast friend begin look oskar eli innocently spend night occasion privy oskar doesnt happens elias caretaker serial killer brutal order desanguinating victim bucket soon oskar realize new friend bite tragedy shock violence eli leave fend desperately stave urge drink fresh blood form delicate new bond oskar remake let right dont wait hope capture magic original gorgeously shot impeccably written fully realize unmatched respect vampire lore acting producer amaze young actors odd recapture brilliant melancholy chemistry astronomical let right think traditional opt endless ultra violence cut favor kiddie friendly rating director tomas alfredson steer line right middle violence brutal horrific dwell leave question kidney display stillness mimic oskar stage stuck moving hope growth begin changes apparent stillness oskar elik oskar grow change man eli stick burdensome fate quiet understate ending haunting whimsical forth times people understanding kind people beg dont miss time silence lambs real chance home oscar gold deserve a+ |
| 11 | 6 | 0.8893 | character, strange, staples, instance, frightened, time, mix, novelized, strong, isi, sleep, misshapen, human, faith, cock, stylization, characters, dont, lippi, phobia | let right heart sweet coming-of-age story unique different defy categorizations swedish adapt john aside lindqvist bestselling book director tomas alfredson dare mix pleasure pain horrify tender let right storyline reveal secret early leave surprise necessarily rare review plot better 12-year-olds oskar kare hedebrant eli klina leandersson meet snowy afternoon jungle gym courtyard oskar house complex outside stockholm young tender attraction apparent right start think relationship headed deep dark secret discover reveal audience repulse curiously fascinate time similar fashion yellow crime scene tape bring close warn support cast completely beholden narrative revolve adorable young couple performance rival ive actor age innocence vulnerability hedebrant oskar tour-de-force admirably carry shoulders leandersson match scene scene line line result literally chills production value stellary technical aspect -- lighting original music johan soderqvist hoyle van hoytema cinematography -- combine perfect synchronization produce hitchcockian tale bring love light dark drama imaginable let right overwhelm choice best narrative feature north american premiere 2008 tribes festival truly well-deserved honor tomas alfredson craft brilliant work art leave shake head wonder |
| 12 | 12 | 0.9088 | dont, rowlands, department, room, police, someones, ride, mind-blowing, expense, 2010, deep, plot, electric, payback, constantly, attempt, muse, relaxed, decent, 1408 | tomas alfredson let right original dark twist gory fantasy special hard classify merely exercise style brilliant piece amoral storytellings characters action defy logic common sense dont wanna spoil moment ll mean revenge moment story happens remind fantasy tale oskar kåre hedebrant year-old bully boy befriend develop innocent crush new neighbor eli klina leandersson happen vampired twist tale revenge pubescent love visual flair swim pool scene classic creative direct impressive performance young pair protagonists hollywood course didnt waste time announce upcoming remake lazy read subtitles likely remake turn pg-13 order money fill moral value prudish parent let kid watch dont vampired year ago ignore future bomb enjoy original youre treaty |
| 13 | 57 | 0.896 | compilation, dvds, unquestionably, nanchuka, penikett, incredibly, restriction, choppy, whisky, lot-, bonkers, ducournau, marvellous, shift, icy, represents, control, hauser, entertains, shortcoming | people review idb focus sweet friendship year old human vampire leads huge element sweet story childhood friendship godfather story father concern worry kids true descriptions miss point think seen truth let right dark measure brilliant story tell week ago disturb echo stay refuse enjoy prior knowledge impossible review properly disclose plot elements warn spoilers ahead comment said character hokan key story end lead believe oskar step place fact reveal final scene dark nightmarish core blossom friendship witness prior offer redemption oskar path relationship eli takes håkan butcher boy old oskar food elik story storytelling high order dark whisper hint suggest questions allude play mind answered hokan pick victim presumably elias preference elias interest oskar elias true nature time briefly glimpse true physical age fact late middle age roughly age håkan gift oskar moments brutal slaughter child tormentors entranced glamour oskar happy happy happily sunset oskar presumably deaden soul destroy husk person hokan butcher fellow human provide eli food puzzle people think sweet fact conclusion utterly chilling oskar happily swap commonplace misery childhood bully fate truly hell earth happily smile shoot doesnt clue superlative heartily recommends tale succeed suggestion play imagination truly addition vampire cannon variation bram stokers brilliant original |
| 14 | 72 | 0.9758 | lowlands, chests, rapid, condoms, christos, airbender, doubt, watched, sheet, 13-year, portion, onethe, dimension, bleeding, brents, antecedent, la, sunlight, well-to-do, autobiographical | fond vampire genre artistic poetic profound explore nature evil world child pure evil pure goodness somewhat discernible achieve astound array contrast allow evil coexist thought-provoking leave audience yearn read novel short gems need remake accord dont need wait 2010 watch |
| 15 | 1 | 0.5361 | story, short, uncertainty, role, watch, fun, grotesque, pretty, theme, time, resistance, handel, trade, purikitpanya, behaviors, victorian, home, ameliorate, look, dubbed | zombies zombies cheap outta original idea moves mandatory width train play beautiful girl sucker long-haired woman short skirts babe morality story ethic means played |
| 16 | 103 | 0.9617 | diverse, answer, nod, outdo, roadwork, depths, hath, pm, surrealism, pruitt, ambiguous, laboratory, outside, vampired, videos, slow-buring, stormy, infernal, filmophile, early | different zombies army undead create industrial mishap result overwhelm chaos danger address morality root cause personally love pretty woman enjoy leg beautiful hair girl hat |
| 17 | 42 | 0.9646 | light, decisions, monikered, harvey, chockful, excitement, sleeping, manners, behavioral, acted, large, launcher, occupant, shockers, vagina, stared, efficiency, spoilerish, westerner, maximum | train susan treat mind eyes look capitalism vs humanity man vs empathy remember time enjoy foreign act excellent effeminate young man story original definitely eye delicious cheerleader |
| 18 | 136 | 0.9722 | actors, humble, compress, chat, close-ups, villains, 3½, thrilling, let, oft, devil, staple, conscience, expected, digitally, relevant, xanthe, takami, romance, minimal | dont youths fun action-packed dangerous elements innovative pretty girl importantly successful yup hollywood remake entry series redundant unnecessary inferior whitewash woman western taliban style ugly hollywood remakes paul fig jar jar abrams negotiate cut speak |
| 19 | 4 | 0.9183 | park, metamorphosis, talked, convict, ism, exit, attacks, abandonment, imbalanced, truth, begun, foggiest, cops, bolt, impeccably, limbless, people, chase, money-obsessed, atv | zombie plenty thistle cheerleader splendid gods gift debut story peter end |
| 20 | 72 | 0.8413 | lowlands, chests, rapid, condoms, christos, airbender, doubt, watched, sheet, 13-year, portion, onethe, dimension, bleeding, brents, antecedent, la, sunlight, well-to-do, autobiographical | ve guess director saw end inventive filmmaker work industry james wan brilliantly retro day deliver extremely stylistic visually strike stand tall classics virtually sex gore curse conjuring earn r-rating scare set 1971 conjuring focus marry paranormal researcher ed patrick wilson lorraine vera farmiga warren lecture college interest case theyre think retirement cue perron family parent roger iron livingstone carolyn lili taylor scare live live daughter claim evil rhode island home doesnt warrens discover perrons torment supernatural wanting short conjuring terrify ive seen trying erase torture porn crowd prove difficult director saw doubt finally look pg normally frown junkiest despite sex gore swearing james wants late slap r rate youre wonder frighten think mpa speak behalf day climax middle turn follow doesnt affect conjuring comic relief brilliantly place bring climax chance build fear startle wan understand psychology tension build suspense mere scene construction obviously note exorcist amityville inspiration derive real case file warren famous case date straying ironic style famous cabin woods surface story inventive assure work creative recent memory ive love james wan manage unoriginal haunted-house- possession story cases weave scares pacing sound design camera work describe precise james wan cinematographer john leonetti responsible look insidious fresh visual style unlike include wide shot movement cinematography absolutely stunning rely shaky cam realism leave unique feel separate visually narratively relate toone issue ive recent genre reveal demon insidious sinister example hook story fear unknown filmmaker audience wan definitely learn insidious conjuring apparition arendt reveal audience theyre far focus letting use imagination truly horrifying think filmmakers note mrs wan want complaint isnt wrong market screw trailer conjuring reveal scares avoid trailer command james wants twitter account hard miss tv spots wish fresh mind havenet trailer want stay marketing ultimately overall production value allow conjuring stand rot genre act impressive practical effect perfect classic feel wan time movies must-see summer rating 10let twitter thejoshl think conjuring |
| 21 | 64 | 0.7754 | vanya, reverse, cross, slowly, pig-boy, revenge, gory, pumpkin, rosary, mouth, olin, alaska, crape, opt, likeable, cody, stroke, drum, blasting, persistently | dont summon devil dont priest lucky conjuring preview screen brightest 2013 totally cold trailer story plot turn scary seen conjuring nail-biting hiding-behind-hands youve disappoint paranormal activity insidious likely deliver area failed tell supposedly true story paranormal investigators aim rid family property suspect supernatural visitations disprove turn creaky floorboard slam doors tackle head-on leap faith require buy theme youre okay play pretty genre confines particular incident bring deal describe surround spirit malevolent hide public fairly amityville-like circumstances family new house discover basement sealed boarded-up doorway surprise follows decide look basement surprise james wan manage tire theme haunt possession revive convincingly strange kind truly tension terror enjoy sinister recently equally scary lose bite end conjuring tempo fairly satisfy conclusion turn comical middle minutes fire cylinder head finale intentional lure false sense security work beautifully youre type poo-poo genre general canst suddenly convert believer enjoy classic exorcista amityville poltergeist guarantee conjuring wont disappoint |
| 22 | 47 | 0.7383 | morally-reprehensible, invitation, ism, heartless, martinet, meek, admission, transfusion, cheol-so, outing, look-alike, cross, enthralled, mushroom, finest, elves, gratitudes, silk, assistant, sullen | ism avid fan lately ive think isnt scare sinister skin appreciate james wants love saw insidious damn modern ghost story review state kinda lose momentum final act conjuring better scarier tense insidious ism gonna limb years classic rule making bad guy fully audience plus suspense tell throw water bottle screen sheer terror scare finally happened fake jump scares earn r rate blood sex profanity terror build scary masterpiece script acting direction style tone notch wants camera-work far movies choice set stroke genius feel authentic real story set present wouldnt effective scary style making costuming hair style throwback exorcist amityville extremely minor flaws length repetitiveness resolution happen fast dont care general flaws perfect havenet james wants conjuring ive time time scare badly dont miss amaze theater view experienced |
| 23 | 2 | 0.9494 | scare, mysterious, recovery, gleeful, feel, place, favorite, styx, wonderous, look, chance, based, deteriorate, character, adage, clich, time, seek, story, support | key conjuring freshness evidence ream fan argue internet site new table fan hungry time conjuring right splinter deal admire day age stand tall proud ream remakes sequel teen friendly slasher haunt multiplex frequency days free gore sex automatically alienate portion lustful member fan based serve appetising scare cauldron suspense deliver plenty particular table forget based true story tag kind irrelevant new technological age sell gimmick mean story true play bit regardless hoax charge embellishments buy premises commit scary story director james wan reward compliant story essentially base investigation early seventy paranormal specialist ed lorraine warren aid perron family victim dreadful supernatural event rhode island home wan build deftly let perron family live believe dream home start happen wan build slow instances create palpable sense dread camera work intelligent moment maximum impact yearn warrens involved kill boo-jump scares place care marry superbly mount tension naturally clich convention haunt house strange smells creaky doors ominous cellar supplement wants talented knack scare effective production designs mysterious bruise literal leg pull breath hold game hide seek bona fide pant soil moments conjuring lesson sustain unease finale unleashed script devoid cheese pointless filler refresh sub-genre suffer problems joseph bishara musical score absolute nerve shredder refresh accompaniment doesnt resort telegraph shriek tell afraid overwhelm scene john leonetti cinematography gothic textures house outside lakeside farmhouses strong lead cast vera farmiga patrick wilson lili taylor ron livingston trump met critical box office success conjuring justify reputation superb haunt house |
| 24 | 12 | 0.9914 | dont, rowlands, department, room, police, someones, ride, mind-blowing, expense, 2010, deep, plot, electric, payback, constantly, attempt, muse, relaxed, decent, 1408 | conjuring high class hard scare care character story compel feel interest time based true life events ed lorraine warren paranormal investigator set help family terrorize demon terrify case live hadnt share love place 70 agree completely director james wants point view impossible set present example teenage daughter haunt family picture demon diphones post post instagram basically demon turn victim human bad guys isnt perfect thought plot hole scare jump seat build tension let character start craziness set 70 feel 70 slow zoom style wouldnt expect modern believe james wan outdo ill far scary ive life thank james wan luck fast furious d recommend look guess feel audience feel watch exorcist dont look poltergeist time |
| 25 | 21 | 0.991 | dry-ed, rashidi, immobile, salinsageonui, fun-loving, connect, necessary, cold-hearted, saw, operating, lifestyle, faceless, potholing, official, click, happens, high, copious, abstract, cases | pit human horrendous extra terrestrial end cheap imitation aliens series pitch black stand fine piece sci-fi excellent favorite aspect lighting beautifully employ different colors shade intensity light set mood lend unique feel different normal light generally subject vin diesel bring character life excellent manner skillfully avoid routine portrayal harden criminal riddick diesel character personal journey thankfully vin doesnt drop ball remainder cast exception talented gorgeous claudia black unknown turn marvelous performances animate diverse character unique quirk mannerisms pitch black perfect example resource excessive budget special effect adequate time mean sole focus high budget blockbusters use science fiction medium tell engaging provoke story tell mediocre story use flash science fiction |
| 26 | 6 | 0.9917 | character, strange, staples, instance, frightened, time, mix, novelized, strong, isi, sleep, misshapen, human, faith, cock, stylization, characters, dont, lippi, phobia | pitch black survival story survive hostile alien world hostile enemies task difficult near enemy survive group plot pitch black usual time different variations shine well-written characters group consist different people interest jacks boy secret fryd pilot hard time conscience johns bounty- hunter drug-habit imams holy man face fact god cruel riddick convict murderer learn value characters start live arendt paper genre start care characters reddick feel odd suppose bad guy line light pitch black think character familiar work casting rarely actor fit role radha mitchell suitable fryd cole hauser bring right cruelness sense responsibility johns impressive work vin diesel job riddick hand reddick creepy definitely dangerous deep character suppose survive pitch black friend close enemy closer |
| 27 | 95 | 0.702 | stance, bowl, mike, uncover, deadi, ted, marsha, draining, meager, holden, dalle, justify, derrickson, thats, easy-to-spot, kuroki, bliss, character, weakness, 70 | admit fan serenity reddick chronicles bite predetermine anticipation flip ism vin geisel fan somewhat mono-dimensional range characters embodiment riddick fit perfectly story play pace encumber gratuitous special effect fit cgi appropriate glare technical defect story start walk breathe air slight concern difference atmosphere microbe common flaw gloss stories canst harp detail radha mitchell smoking hot ridiculous frivolous tough season interplanetary merchant marine ships captain character fry nail perfectly remind andrea start hungarian assassin gilda tigers lady appear drop check heartbeat digress cases trouble grant star pitch black |
| 28 | 91 | 0.7345 | farmhouses, cursory, reopen, hot, 1986, zombie, colourings, reviews, distill, finesse, purchased, fearful, bombarding, diverge, bin, booth, loose, engage, grasped, position | doubt excite satisfy ive years plot print banal- ship crash desert planet suns survivor adjust landscape darkness fall monster appear pilot fryd moment cowardice descent atmosphere jettison passengers charge group enlist help convict murderer reddick lead darkness escape ship surgically enhance eye dark simple character complex three-dimensional conflict trait whoas whoas bathe performance uniformly superb radha mitchell fry steel leadership overcome fears prevent bloodshed cole hauser man reddick custody agendum idiosyncratic standards belong vin diesel reddick magnetic screen presence ive years face shadow doesnt deal draw attention draw attention speak moving diesel doesnt trivialise character easily heart gold reddick mean vicious man approach ship let glimpse tentative step care self care technically light effect excellent end spectrum overbright triple sunlight pitch darkness special effect riddick monsters point view add suspense sound effect monster fly ultrasound monster anatomically plausible suitably frightening editing tight jar time literally pad fades time re-orient open shoot end credit wit scene line dialogue single camera shoot important time understand ally caveat science solar system model impossible planets revolve suns vice versa doesnt affect human story havenet point |
| 29 | 146 | 0.973 | cody, flinch, act, resurrect, intensify, drench, solidly, depressing, tatiana, soundproof, linger, nosferatu, opening, happen, persist, se7en, special, explosion, favors, horror- | open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain |
In [120]:
hdp_most_representative_docs = get_most_representative_docs(hdp_dom_topics_final, n_topics=20)
hdp_most_representative_docs.style.applymap(color_green).applymap(make_bold)
Out[120]:
| Probability_Contribution | Topic_Keywords | Most_Representative_Document | |
|---|---|---|---|
| Topic_Number | |||
| 0 | 0.3079 | room, story, 1408, character, hard-pressed, play, time, feel, irreversible, work, new, mist, similar, cusack, edition, ripped-to-shreds, perversely, noise, progress, bubblegum | anthology sum parts particular set stack novice young man trouble past leave live monks karma catch interesting idea lack execution bite bore end ward injure teenager spend night hospital room brain dead old man respirator slate life support day spooky violent twist ending backpackers favorite bunch young hitchhiker pick trucker cargo dangerous id love salvage dishonest car saleswoman terrify night vehicle return angry customer son disappears gruesome ghost story feature karma main theme end funny subversive tale spooky long-haired ghost girl east frequently 10+ years phobia rocky start overall entertain vary mix thai |
| 1 | 0.2765 | story, short, uncertainty, role, watch, fun, grotesque, pretty, theme, time, resistance, handel, trade, purikitpanya, behaviors, victorian, home, ameliorate, look, dubbed | poke fun general youre look cradle scare lady-friend watch southpark britney spears episode parallel matt stone trey parker draw modern culture glorify celebrity ancient sacrifice god ensure harvest same- create reality tv type scenario main character walk step clich joker jock slut nerd virgin elaborate version kind sacrifice genre supply endless stream blood satisfy ancient need character clich notice falling cast role human tip hat writers joss clever beasts pure horror- thats point enjoy commentary narrative develope didnt head happening edge interest end point sit think wow fun definitely fun watch |
| 2 | 0.3555 | scare, mysterious, recovery, gleeful, feel, place, favorite, styx, wonderous, look, chance, based, deteriorate, character, adage, clich, time, seek, story, support | plot solid entertain mean want new watch genre- entertain lore improve story flashback creative storytelling gem minute doesnt provide time viewer understand evil descend family tell bit piece owner run funeral parlour home owner dagmar hide sell body house build ancient evil leave guess heck rest story old boiler downstairs minute screen time flesh sordid past leave satisfy true understand house evil past leave gore fest jump scare new old fashion star rating |
| 3 | 0.582 | prof, laurier, dont, story, minami, rides, think, icon, le, mr, texas, peasant, marriage, push, melodramatics, tightly-wired, bewilder, titular, similarities, deceitfully | point mince words brad anderson session ive time intelligent well-written completely unpredictable look didnt notice view edit photography work soundtrack downright creepy recently manage lie awake night dario argentous opera tobe hoopers texas chain saw massacre list include honestly excuse folks doesnt |
| 4 | 0.2166 | park, metamorphosis, talked, convict, ism, exit, attacks, abandonment, imbalanced, truth, begun, foggiest, cops, bolt, impeccably, limbless, people, chase, money-obsessed, atv | massive fan leave completely cold mean spirited cold unbelievably frustrate stupid times plus act involved lack logic place complete lack satisfy pay merit quickly outweighed example lead character think okay wonder people home consent lead bizarre random mobile phone suspension disbelief lose appropriate response doh ending terrorisation expect decent pay instead downbeats nihilistic sign popular modern miss point somewhat experience unpleasant hate work people |
| 5 | 0.9157 | water, scene, funeral, sarsgaard, yoshimi, feel, diner, tight, adage, chainsaws, misunderstand, silk, neighbours, ing, secretary, outer, dark, afterthought, self-sacrifice, reveal | hell ride minute stop look time leave relieved left thats engage interest story watch read review smorgasbord genre positive review doesnt feel means yes storytelling fluid write multiple genre aspect didnt feel place works work feel redundant point genres similar witch necessarily term storyline pace severity terror psychological wailing deliver somewhat complex storyline cinematography breathtaking jun kunimura ridiculous oh-so-good actor point ism point mention asian technically guess lol dont think necessary pigeonhole |
| 6 | 0.3766 | character, strange, staples, instance, frightened, time, mix, novelized, strong, isi, sleep, misshapen, human, faith, cock, stylization, characters, dont, lippi, phobia | admit excellent scare gory stories novice ward backpackers salvage end novices gory ward scary backpackers thrilling salvage scary gory end funny scary phobia bias bias pawn purikitpanya story bite weird improvement phobia phobia end cast 4bia- middle think funny marsha wattanapanich play funny act funny overreacted thats ism phobia vote |
| 7 | 0.8265 | salinsageonui, sewer, idea, plot-heavy, tab, angina, visitors, faces, london-town, jesses, eery, response, decoration, negotiable, produce, meshes, so-so, grizzly, hills, spring | die darks festival japanese chill complex spooky follow nagisa japanese actress base real murder spree place large hotel forty-some year ago excite look forward filming scary begin spooky angina begin ghostly girl vision intensify director cast real hotel actual murder happened inspiration connection hotel grisly murders answer soon havenet asian original ju-on ive remake havenet pretty wasnt expect luckily pleasantly surprised reincarnation grinned original language solid supernatural rise far expectations write nicely inconsistency major midway think figure positive plot twist turn pretty confident figure final minute entire assumption disintegrated leave minor shocked ism rest plot twist coming suppose expect twist set-up trick audience admit work intelligent writing director takashi shimuzu direct ju-on ju-on american remake grudge grudge eye eerie unlike style grudge reincarnation boast subtle spookiness personally menace annoy jump scares numerous shot scene hard forget direction lots creepy shot disturb imagery homemade snuff footage eerie creepy music fit perfectly slow subtle acting-wise id succeed performer pretty believable language barrier think conclusion real kickers surprise creeped entire close overall reincarnation supernatural traditional ghost story throw hotel reincarnation theme youve enjoy slow andor fan japanese id recommend minor problems easily outweigh bad end surprising ism fit tastes couldnt pick foreign flick |
| 8 | 0.9828 | carmen, chainsaw, outcasts, petty, se7en, jolts, girls, 911, hollow, paula, safe, require, nullify, stillness, iranian, suppress, vampire, think, diverse, epilepticmoondancer | think creepy tone vibe atmosphere begin minutes think creepy feel clever element throw wrong bore doesnt artistic original approach classic direction falters mainly slow build isnt entice new build pointless drivel lengthen runtime audience member classic movies pace bad overall forgettable flick isnt worth time semi-retro cinematography look potential maybe delve mystery element revolve town people |
| 9 | 0.4927 | button, person, plot, live, essence, reveal, paste, people, su-mi, jurassic, advice, murder, check-out, hurricane, nights, whilst, states, boast, grainy, genitals | jeepers creepers common 1950 ec comic book past years fact isnt bad plot idea decent expression decent surprise pace stay interesting identity monster stay mysterious youre totally certain suppose plot intelligently grade dialog isnt stupid contrived rend worth cash |
| 10 | 0.3416 | watch, cinematically, repopulate, burned, slapstick, confessions, fault, korea, resist, institute, story, thailand, bright, think, finally, service, wolf, festival, somethings, dispatch | deserve allot praise play norwegian cultural memes visually landscape place folklore troll arose course spice lovely computer graphic needed graphic merge rockumentary style fall categories mix comedy action fake amateur documentary style bad lack bite flow times course drawback apply foreigners norwegian story portuguese girlfriend think okay brief fairy tale joke sheep bridge hard catch appreciated marinate culture time issue cameo appearance norwegian comedians slightly spoil illusion disagree suspect seven norwegians concepts rich despite flaws ll love visual drama norwegians exotic neat offer relief classical hollywood monster depictions curious google theodor kittelsen asbjornsen moed enjoy |
| 11 | 0.7247 | remote-control, benji, shouts, biblical, two-story, produce, scandal, archetype, jungle, shaman, health, acceptable, parallel, udo, lineage, refuge, loser, till, unexpectedly, palmas | confessions savage brutal poignant revenge story seen doesnt start end awe begin japanese classroom final day class spring break remainder event follow fateful day classroom point view switch numerous time different people affect event beginning progress revisit past scene different characters perspective scene monotonous time onion confessions multi-faceted discover re-discover subsequent layer peel backra beautiful wildly sympathy flip-flopped dont adamantly learn character event sympathy bind jump ship point youre totally confessions challenge frustrate add confessions charm wit unrelentingly dark bind incite feeling bleakness suit weak heart subject matt heavy character morally-reprehensible feel wholly somber grey dull blue saturate scene melancholy albeit perfectly-suited soundtrack work infectiously mind horrific scene eye resonate deeply spiel longer closing act top-notch storytelling captivating cinematography gorgeous touch violence blood taste immense effect feel gratuitous japanese cake seen ive far sound convince definitely shoot hard let ounce |
| 12 | 0.3394 | dont, rowlands, department, room, police, someones, ride, mind-blowing, expense, 2010, deep, plot, electric, payback, constantly, attempt, muse, relaxed, decent, 1408 | incredibly cruel unrelenting play single feature divide sections dumplings direct fruit chan hong kong cut direct park chan-wook korea box direct mike takashi japan section dissertation dumplings classify exploitation beautifully produce directed box lavish sets costume atmosphere culture respective country provide new terrify appalling original right cut compare saw hannibal letter franchise theres psycho whoas message murder mayhem dont want detail plot think viewer need experience preconception dumplings plot high squeam factor cut cover familiar premise fun moment ll box artistic endeavour plot reason enthrall couldnt look mostra solid effort genius asian directors |
| 13 | 0.331 | boyle, progresses, hour, catwalk, rarely, trash, post-birthday, higher, stringy, couple, jeonmal, critic, trier, greatness, peace, unnatural, actual, connections, quarantine, allow | caught resolution festival describe doesnt fit sub-genre definitely belong family creative unique ive thoroughly entertaining pace guess entire time conclusion personally canst wait view lead performance essential isolate setting end people love hated love speak fellow audience member dislike dont end loving ll doubt entertained plus provide interest conversation fellow fans think viewers refresh unique quality hard hard modern wont regret star |
| 14 | 0.7917 | tail, resolution, gozu, inexplicable, walls, calmly, titanic, fears, story, fukikoshi, carolyn, lawyer, scene, frantic, theatrical, hillside, people, under-30, oversights, third-wheel | approach adaptation video game extreme trepidation think corker weave catch tv past super mario bros mortal combat resident evil stinkers doom vapid close source materials silent hill open music lift straight game fact entire contain original track game remix score fantastic brood background play game work effectively screens story composite story element games final act monstrous highly original thirty minute guess completely gross violence pretty gruesome clever mixture cgi fully made-up monster add forebode atmosphere feel nervous watch visual absolutely spot ive adaptation close source material comic book past year stray furiously weak opening id resign fact disappointing pleasantly surprise soon main character rose silent hill youre fan game wont disappointed youre hounds ll love creature visuals youre expect deep relationship estrange mother daughter youre wrong theatre highly recommended best ive time |
| 15 | 0.9711 | woods, starkly, e-bay, wood, audience, george, tommy, horrifying, greatness, shower, kiddie, twitchers, exaggerate, limp, unsettling, disturb, terrifying, consecutive, foul-mouthed, basking | description think yeah possession yawn waste evening look positive review decide disappointment original material craft cinematic acting plot reasonably scary suspenseful solid recommend strongly fan trust dont recommendation easily plenty crape gold job |
| 16 | 0.5523 | lawn, twitch, unease, mentally, vampire, ein, wonder, horror-thrillers, jeong-min, estrange, porn, freshment, leaver, professor, spar, provide, obnoxious, intend, depraved, wes | idea boy pleasantly surprised director rec set apartment block spanish gem cast lead luis play superbly fill increase sense unease shock youre witnessing beautifully shoot took apartment block setting end leave think happened far creepiest twist tale ive pleasure watch time highly recommended |
| 17 | 0.7554 | weak-stomached, dimension, abuses, wee, centre, suspects, issue, simple-minded, eager, toro, archetype, choose, plural, puzzle, breathtaking, off-guard, help, gunshot, plan, 80s | ju-on grudge easy america wasnt write review hear hype heaven magazine fangoria rue morgue word mouth finally track la watch chance ju-on chapter story haunt house tokyo suburb begin inexperienced social worker house face face story jump past present chapter focus character time circle unwise enter curse house wind dead haunt spread virus terrible murder place house rage surround act violence spawn evil curses enter house immediately infect haunt follow people home drive near madness drag ju-on bear pass resemblance popular predecessor ringu near frightening bad means butchered mother-ghost kayaks sadako-like crawl black hair face unearthly jerkiness blue-white face startle huge stare eye occasional splash blood ghost son toshio sad frightening appear normal boy pale wide-eyed ghost frighten sequence feature murder woman kayako head black hair peek corner shadow corridor fill security camera head-on shoot crawl attic night beam flashlight illuminate sound effect disturb performance convincingly wasnt scare promise thats happen buy hyper expect pretty ghost story instead ju-on decent straightforward ghost story scary moment fans ring scarier ju-on noble effort asian stories remain ambiguous open-ended leave room sequel chance decide curse grudge stars |
| 18 | 0.8376 | elves, roth, slicks, hoodoo, days, slam-bang, disturbance, et, red-eye, fanatical, shouting, sanitize, hellraiser, keane, heighten, atmosphere, spring, dreamy, jumped, love | seeing session reaffirm truly including filmmakers comparison dont look shining nod changeling session stand effective brood experience dread -- thats things style tension genuine grandiose shining session rely real fear gratuitous material entertain want creep thats soul purpose pretty young gap models trendy mtv-influenced rap metal soundtracks breasts giggle-inducing decapitation effects want mean watch plenty offer short attention spans want draw place fear think session therapy session nightmares story simple complex time worker remove asbestos massive danvers mental hospital slowly unravel secret tape audio recording patient characters background underdeveloped complain -- dont need guy information slow nice brood pace right dont worry clues leave unresolved thats fun weird clues david lynches work twin peaks mulholland drive weird draw direct connection main story actual danvers hospital amaze set structure character disturb minimalistic soundtrack unsettle downright perfect buy cd figure attract play background type actor fine job -- yes david caruso need guy bite impatient maintain sense calm peter mullane steven gevedon josh lucas brendan sexton iii real regular blue collar guys refresh insecure mature rough-edged cast sense guy real eyes tend slack bite place theyre suppose job weeks quiet dread story draw ll absorb completely course appreciate flashy ll agree damn near perfect idb criticize scene jar peanut butter leave floor welllll think consider state consumer think leave care container disposed man people picky strange things implausible story characters action doesnt wreck appreciate atmosphere story didnt end hard understand mind set didnt busy angry sit let present clearly categorize ive seen definitely years scare theres specific scene josh experience basement cause wave tingle goosebumps body exhilarate scare effectively single scene folks canst trash doesnt easy explanation allow cheap voyeuristic thrills didnt session read summary wasnt offer slam-bang entertainment mind stimulate love absorb mysterious wondrous storytellings session definition flawless piece work id |
| 19 | 0.5641 | rationalism, unearthly, declining, meandering, gate, ignite, starting, malice, zack, lost--this, stinker, heisman, rich, real, taper, ls054808375, increase, perplex, rapidly, exist | fan art drama dont genre understand hate true fan basic knowledge cult history genre real joy cabin woods weird precisely genre determined gather idea countless time turn upside-down new ability original totally outwear cliches parody homage time funny-scary visual dialogue one-liners pace twist predictable let bore minutes final revelation time ridiculous awesome screen masterpiece real gem love |
In [121]:
hdp_topic_dist = get_topic_distribution(hdp_dom_topics_final, n_topics=20, hdp_flag=True)
hdp_topic_dist.style.applymap(color_green).applymap(make_bold)
Out[121]:
| Documents_per_Topic | Percent_of_Total_Corpus | Topic_Keywords | |
|---|---|---|---|
| Topic_Number | |||
| 0 | 66 | 0.066 | room, story, 1408, character, hard-pressed, play, time, feel, irreversible, work, new, mist, similar, cusack, edition, ripped-to-shreds, perversely, noise, progress, bubblegum |
| 1 | 58 | 0.058 | story, short, uncertainty, role, watch, fun, grotesque, pretty, theme, time, resistance, handel, trade, purikitpanya, behaviors, victorian, home, ameliorate, look, dubbed |
| 2 | 26 | 0.026 | scare, mysterious, recovery, gleeful, feel, place, favorite, styx, wonderous, look, chance, based, deteriorate, character, adage, clich, time, seek, story, support |
| 3 | 20 | 0.02 | prof, laurier, dont, story, minami, rides, think, icon, le, mr, texas, peasant, marriage, push, melodramatics, tightly-wired, bewilder, titular, similarities, deceitfully |
| 4 | 18 | 0.018 | park, metamorphosis, talked, convict, ism, exit, attacks, abandonment, imbalanced, truth, begun, foggiest, cops, bolt, impeccably, limbless, people, chase, money-obsessed, atv |
| 5 | 16 | 0.016 | water, scene, funeral, sarsgaard, yoshimi, feel, diner, tight, adage, chainsaws, misunderstand, silk, neighbours, ing, secretary, outer, dark, afterthought, self-sacrifice, reveal |
| 6 | 22 | 0.022 | character, strange, staples, instance, frightened, time, mix, novelized, strong, isi, sleep, misshapen, human, faith, cock, stylization, characters, dont, lippi, phobia |
| 7 | 14 | 0.014 | salinsageonui, sewer, idea, plot-heavy, tab, angina, visitors, faces, london-town, jesses, eery, response, decoration, negotiable, produce, meshes, so-so, grizzly, hills, spring |
| 8 | 18 | 0.018 | carmen, chainsaw, outcasts, petty, se7en, jolts, girls, 911, hollow, paula, safe, require, nullify, stillness, iranian, suppress, vampire, think, diverse, epilepticmoondancer |
| 9 | 16 | 0.016 | button, person, plot, live, essence, reveal, paste, people, su-mi, jurassic, advice, murder, check-out, hurricane, nights, whilst, states, boast, grainy, genitals |
| 10 | 20 | 0.02 | watch, cinematically, repopulate, burned, slapstick, confessions, fault, korea, resist, institute, story, thailand, bright, think, finally, service, wolf, festival, somethings, dispatch |
| 11 | 8 | 0.008 | remote-control, benji, shouts, biblical, two-story, produce, scandal, archetype, jungle, shaman, health, acceptable, parallel, udo, lineage, refuge, loser, till, unexpectedly, palmas |
| 12 | 20 | 0.02 | dont, rowlands, department, room, police, someones, ride, mind-blowing, expense, 2010, deep, plot, electric, payback, constantly, attempt, muse, relaxed, decent, 1408 |
| 13 | 16 | 0.016 | boyle, progresses, hour, catwalk, rarely, trash, post-birthday, higher, stringy, couple, jeonmal, critic, trier, greatness, peace, unnatural, actual, connections, quarantine, allow |
| 14 | 12 | 0.012 | tail, resolution, gozu, inexplicable, walls, calmly, titanic, fears, story, fukikoshi, carolyn, lawyer, scene, frantic, theatrical, hillside, people, under-30, oversights, third-wheel |
| 15 | 16 | 0.016 | woods, starkly, e-bay, wood, audience, george, tommy, horrifying, greatness, shower, kiddie, twitchers, exaggerate, limp, unsettling, disturb, terrifying, consecutive, foul-mouthed, basking |
| 16 | 10 | 0.01 | lawn, twitch, unease, mentally, vampire, ein, wonder, horror-thrillers, jeong-min, estrange, porn, freshment, leaver, professor, spar, provide, obnoxious, intend, depraved, wes |
| 17 | 8 | 0.008 | weak-stomached, dimension, abuses, wee, centre, suspects, issue, simple-minded, eager, toro, archetype, choose, plural, puzzle, breathtaking, off-guard, help, gunshot, plan, 80s |
| 18 | 10 | 0.01 | elves, roth, slicks, hoodoo, days, slam-bang, disturbance, et, red-eye, fanatical, shouting, sanitize, hellraiser, keane, heighten, atmosphere, spring, dreamy, jumped, love |
| 19 | 8 | 0.008 | rationalism, unearthly, declining, meandering, gate, ignite, starting, malice, zack, lost--this, stinker, heisman, rich, real, taper, ls054808375, increase, perplex, rapidly, exist |
In [122]:
hdp_topics = []
for topic_id, topic in hdp.show_topics(num_topics=20, formatted=False):
topic = [word for word, _ in topic]
hdp_topics.append(topic)
In [123]:
hdp_cm_umass = CoherenceModel(topics=hdp_topics, corpus=corpus, dictionary=dictionary, coherence='u_mass')
print('The U_MASS coherence measure of the HDP Model is: {}'.format(hdp_cm_umass.get_coherence()))
The U_MASS coherence measure of the HDP Model is: -17.870792618102282
In [124]:
hdp_cm_cv = CoherenceModel(topics=hdp_topics, texts=all_reviews_text, dictionary=dictionary, coherence='c_v')
print('The C_V coherence measure of the HDP model is: {}'.format(hdp_cm_cv.get_coherence()))
The C_V coherence measure of the HDP model is: 0.6891603871535945
In [80]:
prepared_hdp = pyLDAvis.gensim.prepare(hdp, corpus, dictionary)
display(pyLDAvis.display(prepared_hdp))
LSI (Latent Semantic Indexing)¶
Cf. http://lsa.colorado.edu/papers/JASIS.lsi.90.pdf¶
In [81]:
mod_list_lsi_umass, coh_list_lsi_umass = compute_coherence_values(LsiModel, dictionary, all_reviews_text, max_topics=20,
corpus=corpus, topics=False, transformed_vectorizer=False, tfidf_norm=False,
min_topics=2, stride=3, n_top_words=False, measure='u_mass',
lsi_flag=True)
mod_list_lsi_cv, coh_list_lsi_cv = compute_coherence_values(LsiModel, dictionary, all_reviews_text, max_topics=20,
corpus=corpus, topics=False, transformed_vectorizer=False, tfidf_norm=False,
min_topics=2, stride=3, n_top_words=False, measure='c_v',
lsi_flag=True)
Coherence of model with 2 topics has been computed Coherence of model with 5 topics has been computed Coherence of model with 8 topics has been computed Coherence of model with 11 topics has been computed Coherence of model with 14 topics has been computed Coherence of model with 17 topics has been computed All coherence values have been computed for the <class 'gensim.models.lsimodel.LsiModel'> using the U_MASS measure Coherence of model with 2 topics has been computed Coherence of model with 5 topics has been computed Coherence of model with 8 topics has been computed Coherence of model with 11 topics has been computed Coherence of model with 14 topics has been computed Coherence of model with 17 topics has been computed All coherence values have been computed for the <class 'gensim.models.lsimodel.LsiModel'> using the C_V measure
In [82]:
best_num_lsi_topics = show_best_num_topics('LSI', coh_list_lsi_umass, coh_list_lsi_cv, max_topics=20, min_topics=2, stride=3)
print('The most coherent number of LSI topics to use is: {}'.format(best_num_lsi_topics))
The most coherent number of LSI topics using the U_MASS measure is: 2 The most coherent number of LSI topics using the C_V measure is: 5 The most coherent number of LSI topics to use is: 5
In [83]:
lsi = models.LsiModel(corpus, id2word=dictionary, num_topics=best_num_lsi_topics, chunksize=1000)
In [116]:
#lsi_topics_matrix = lsi.show_topics(formatted=True, num_words=20)
In [117]:
#lsi_topics_matrix = np.array(lsi_topics_matrix)
#lsi_topics_matrix.shape
In [118]:
#lsi_topic_words = lsi_topics_matrix[:,1]
#for i in lsi_topic_words:
# print([reg_ex.sub('', w) for w in i.split() if len(reg_ex.sub('', w)) > 0])
# print('\n')
In [84]:
lsi_df = get_topics(lsi, best_num_lsi_topics)
#print(lsi_df)
lsi_df
Out[84]:
| Topic # 01 | Topic # 02 | Topic # 03 | Topic # 04 | Topic # 05 | |
|---|---|---|---|---|---|
| 0 | story | sarah | people | water | story |
| 1 | time | juno | battle | ring | original |
| 2 | character | friend | story | dark | room |
| 3 | people | cave | world | original | work |
| 4 | think | subconscious | royale | yoshimi | beast |
| 5 | feel | action | school | watch | vampire |
| 6 | watch | deep | youth | scene | ghost |
| 7 | scene | leave | high | work | 1408 |
| 8 | end | stab | island | feel | battle |
| 9 | dont | conscious | kill | world | watch |
| 10 | work | level | idea | girl | french |
| 11 | plot | people | juno | ikuko | school |
| 12 | look | kill | sarah | think | character |
| 13 | want | event | videogame | time | scene |
| 14 | doesnt | dreamer | hate | beast | remake |
| 15 | play | wreck | love | child | le |
| 16 | love | husband | young | nakada | creature |
| 17 | director | story | teenagers | young | frontal |
| 18 | start | run | die | form | mike |
| 19 | genre | death | problem | people | genre |
In [85]:
lsi_hellinger_topic_distances = get_most_similar_topics(lsi, lsi_df, lsi_flag=True, num_topics=best_num_lsi_topics)
pretty_print(lsi_hellinger_topic_distances)
Hellinger Distances between topics are: ______________________________________________________________________ (0.5568771924831382, 'Topic # 01', 'Topic # 05') (0.5564700053205395, 'Topic # 01', 'Topic # 04') (0.5561245367232094, 'Topic # 01', 'Topic # 03') (0.5557440151483, 'Topic # 01', 'Topic # 02') (0.02155843423777627, 'Topic # 03', 'Topic # 05') (0.021381967767978346, 'Topic # 04', 'Topic # 05') (0.01904421110593379, 'Topic # 02', 'Topic # 05') (0.01822447869120819, 'Topic # 03', 'Topic # 04') (0.01618622518404863, 'Topic # 02', 'Topic # 04') (0.015231272000547094, 'Topic # 02', 'Topic # 03') ______________________________________________________________________
In [86]:
lsi_dom_topics_df, lsi_dom_topics_final = get_dominant_topics(lsi, corpus, all_reviews_joined, lsi_flag=True)
In [87]:
#lsi_dom_topics_final.style.applymap(color_green).applymap(make_bold)
lsi_dom_topics_final_first30 = lsi_dom_topics_final.head(30)
print('Note: For the LSI model, the (un-normalized) topic-probability with the largest magnitude is shown! \n')
lsi_dom_topics_final_first30.style.applymap(color_green).applymap(make_bold)
Note: For the LSI model, the (un-normalized) topic-probability with the largest magnitude is shown!
Out[87]:
| Dominant_Topic | Probability_Contribution | Topic_Keywords | Original_Text | |
|---|---|---|---|---|
| Document_Number | ||||
| 0 | 0 | 4.5363 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | coherent plot tell non-straightforward fashion open interpretations leave require audience fill-in open end possibly filling-in require viewings considerable think sense watch wailing time impression kind watch ponder read watch attempt people interpret explain different conclusion coherent underlie plot matt shift piece puzzle attempt recreate coherent narrative piece fit incompetence underlie story possibly intentionally possibly fundamentally inconsistent optical illusion escher drawing appear describe physical object fact dont physical sense accordingly enjoy boil content technically well-made cinematography acting costumes plot sense tell deceptive lure think plot sense matt sufficient thought leave lingering irritate feel dissatisfaction confusion maybe precisely point tell story purpose instill audience feel confusion face sequence event sense life times |
| 1 | 0 | 2.004 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | spoiler review extraordinary hour technically-perfect humanistic fine writer directors business hauteur saw devil cipher close analog suggest david lynches 2001 mulholland drive technically perfect humanistic suspense opus captivate length walk theatre shake head wonder saw film-makers understand secret story tell present story viewer feel share experience protagonist tell story viewer going provide explanation wailing means re-quote writer director numerous interview enigmatically begin wonder nature god -- recommended levels entertainment puzzled clinic engage wont let |
| 2 | 0 | 9.9919 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | wailing open quote bible easy forget fact watch certain point clear purpose quote served warning religious person scare wailing work religious parables overcome lengthy run time tonal imbalanced deliver funny truly terrify mysterious strange small village south korean village plague sickness police think wild mushroom blame police officer jong-goo think stranger meet woman information mysterious man slowly begin fall rabbit hole consume life daughter start sign sickness personal figure trust avoid director hong-jin nap struggle tonal balance time pitch perfect time place tone struggle stay consistent primarily funny brief burst depravity violence comedy work laugh numb times horrific imagery sudden effective balance lose act comedy slapstick fit rest character over-the-top act exaggerate different introduced middle section moment unintentional humor example man strike lighten caller id shaman phone shaman fortunately balance act smartly switch act intense terrify moment comedy offset terrify act start unfold audience gain new appreciation rest start reinterpret certain scenes problem scene sense ended feel natural fit story progressed require end sense hour minutes middle portion resolved shaman screen time perform ritual end important zombie scene feel awkward didnt fit rest want zombie grow popularity scene bring numb random character serve purpose rest despite scene previously mention overacting act generally speak fantastic father task solve happen daughter convey terror hatred build intense persona carry priest train advice father equally effective bring concern innocent quality terror ultimately happen young daughter sick shines channeling inner linda blair emphatically deliver horrible dirty line child performance truly terrify watch hatred eye slowly end religious overtone clearer locusts attack individual white black suggest characters true nature scene truly shines slowly unveil real nature certain characters minute change perspective things minute switch determine trust reveal realize detail setup reveal twist lead ultimately putt faith wrong people wailing 2016 directed hong-jin na screenplay hong-jin na starring woo-hee jeong-min hwang so-yeon range won krak run time hour minute |
| 3 | 0 | 3.1264 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | hell ride minute stop look time leave relieved left thats engage interest story watch read review smorgasbord genre positive review doesnt feel means yes storytelling fluid write multiple genre aspect didnt feel place works work feel redundant point genres similar witch necessarily term storyline pace severity terror psychological wailing deliver somewhat complex storyline cinematography breathtaking jun kunimura ridiculous oh-so-good actor point ism point mention asian technically guess lol dont think necessary pigeonhole |
| 4 | 0 | 3.4346 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | want start review completely jump scares play integral movies rely support story leave displeased wailing special jump scare instead viewers story rich mystery character struggles dark atmosphere design amazingly craft scene blood run cold addition carry subtext leave viewer question evidence interpret feel director time research real experience possible wailing bite bore slow burner construct modern running time th 36min patient man pay end genre dead look cash grab mainstream wide release hide gems |
| 5 | 0 | 6.1234 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | ism biased set city live work oxford street piccadilly circus pass morning usually teem crowd people completely send shiver spine usually watch locate nondescript midwestern village easy detach event unfold screen occur place home entirely new sense reality previously unaccustomed judge days entirely merit easy arrive conclusion fantastic achievements coming-of-age sort director danny boyle canst mtv-inspired vanity beach self-consciously trendy posture trainspotting appeal shame initially expect days similar treatment thankfully fear prove unfounded discard straight open sequence effortless fearsome rest joy terrify joy joy nonetheless true minimalism effective overblown bravado definitely true scene complete silence entire metropolis grainy picture serve add documentary-style quality situation real bear definitely wise choice digital video occasionally meet people think days wasnt scary gory people tell 2001 boring memento confusing ignore didnt understand purpose confuse change pace feel distract personally regard important reviewer point succinct point scarier end world world repopulate maniacs think real days lies 28 days romero zombie flick yore ultimately allegory day live age constantly confront violence medium ape right start violence beget violence humanity face uncertain future applaud danny boilers bravery days undoubtedly commercial risk majority cinema-going public prefer escapism word caution remember rage human-made disease allegory masterpiece time days misunderstand considerable people doubt history classic perfectly sum confuse era live didnt advisable days chance haunt experience look right angle danny boyle year leave hope hell continue |
| 6 | 0 | 1.9574 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | virus rage virus infect person mad extreme rage hungry blood day outbreak london cause entire britain dead evacuate leave blood-thirsty infect population handful solitary normal persons civilization halt society destroy limit survivor fight existence frequent attack vicious victims sounds familiar 28 days classic horde zombie biohazard movies touch art danny boyle able bring focus extensive special-effects-controlled gory action sequence infect normals heavy background music theres tinge sadness emptiness helplessness consider london scene background music theres electrify action gore describe picture life condition talk mistake loophole couldnt weaken tight-gripping storyline achievement viewer stay neutral place virus talk metaphor rage social disease flick person harsh critic zombie watch stars mention billian murphy awesome |
| 7 | 0 | 1.7303 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | ism amaze negative review think succeed level artistic atmospheric pace sympathetic characters fantastic climax music nicely eerie open scene london street worth price admission ism stubborn viewer normally benefit early critical buzz manner excuse time ism completely impressed incidentally think interest day inspire knockoffs knockoffs 28 days apparently owe john wyndham classic disaster novel day triffids thats things highly recommended |
| 8 | 0 | 2.5955 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | 2003 state-side release danny boilers 28 days advertise chockful scare-fest didnt day ago gotta feel somewhat embellishment promoters environmental terrorist attack lab contain diseased chimp infect rage virus unwittingly let loose plague lay waste england rest society days title cut abandon london coma-tized bicycle courier jim cillian murphy effective wake stasis hospital search london sign life rescue rage zombie couple survivalists lovely naomi harris follow place place alive meet man daughter brendan gleeson terrific megan burns refuge london-town record message paradise salvation track frank man shortwave radio feel meditation happen people reduce low elements friend tell movies run zombie inspire zombie remake dawn dead dawn pretty full-throttle action hybrid start finish play emphasis creature lucky survivors disturb lesson nature survival interest disturb flick sell wrong |
| 9 | 0 | 6.7229 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | key sci-fi genre alive cinemas material technique filmmakers present competent average creative havenet havenet style form george romero prime era film-making bring forth memorable trilogy time genre consider romero list director included important source independent 1968 inventive result masterpiece lacklusters director danny boyle author alex garland cook yarn similar romero wouldnt satisfying theyve essential success days later- idea practice years turn fresh audience feel repelled excited terrified nauseous enthralled wont leave feel weave complete hack work boyle team set freshen zombie string precise term zombie movie- hear living-dead utter hear infected new word people rage infect britain monkey virus let loose island begin infectious spread cut man jim lie hospital bed wander abandon street view tear fragment society boyle implement atmosphere heavily action chase scene indication dedication details jim soon survivors include selena naomie harris father daughter brendan gleeson megan burns hear salvation radio decide brave military outpost thats true salvation outside military typos boyle producer andrew macdonald spellbind thats proper terminology adaptation trainspotting craft par arguably toppings story characters captivate care plight jim companions photography anthony dod mantle striking digital photography blair witch use non-professional camera equipment add proper shade needed shoot composition different edit chris gill quick expect attack scenes fast infect throw blood scream on-looker new infect transform seconds result follow mark tildesley production designs john murphys music evoke haunting evocative mood mundane scenes acting consider actor well-known believable script ism days cup tea science fiction fan immediately hear preview tv add dismiss phooey rubble borrow video-store point view similarity romero military scene remind day dead chain zombie practical reasons supermarket scene unneeded consider satirical reverence dawn dead understand boyle isnt 100 original point genres history credible job different tone different set country different envelop view scene structures overall days construct execute sci-fi youve seen sci-fi youve combined sense modern audience fascinate |
| 10 | 0 | 5.2701 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | let right vampire seen smarter scary nuanced doesnt feel thriller feel literature detail bizarre misadventure pair pre-teen star cross lovers androgynous vampired phenomenal regard detail young oskar kare hedebrant life spot stuck incredibly painful period post-childhood pre-adolescence oskar aware girls idea contend small age brutalize boy result terribly collect news clip violent crime let rage day strange young girl eli klina leandersson appear playground fast friend begin look oskar eli innocently spend night occasion privy oskar doesnt happens elias caretaker serial killer brutal order desanguinating victim bucket soon oskar realize new friend bite tragedy shock violence eli leave fend desperately stave urge drink fresh blood form delicate new bond oskar remake let right dont wait hope capture magic original gorgeously shot impeccably written fully realize unmatched respect vampire lore acting producer amaze young actors odd recapture brilliant melancholy chemistry astronomical let right think traditional opt endless ultra violence cut favor kiddie friendly rating director tomas alfredson steer line right middle violence brutal horrific dwell leave question kidney display stillness mimic oskar stage stuck moving hope growth begin changes apparent stillness oskar elik oskar grow change man eli stick burdensome fate quiet understate ending haunting whimsical forth times people understanding kind people beg dont miss time silence lambs real chance home oscar gold deserve a+ |
| 11 | 0 | 4.4798 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | let right heart sweet coming-of-age story unique different defy categorizations swedish adapt john aside lindqvist bestselling book director tomas alfredson dare mix pleasure pain horrify tender let right storyline reveal secret early leave surprise necessarily rare review plot better 12-year-olds oskar kare hedebrant eli klina leandersson meet snowy afternoon jungle gym courtyard oskar house complex outside stockholm young tender attraction apparent right start think relationship headed deep dark secret discover reveal audience repulse curiously fascinate time similar fashion yellow crime scene tape bring close warn support cast completely beholden narrative revolve adorable young couple performance rival ive actor age innocence vulnerability hedebrant oskar tour-de-force admirably carry shoulders leandersson match scene scene line line result literally chills production value stellary technical aspect -- lighting original music johan soderqvist hoyle van hoytema cinematography -- combine perfect synchronization produce hitchcockian tale bring love light dark drama imaginable let right overwhelm choice best narrative feature north american premiere 2008 tribes festival truly well-deserved honor tomas alfredson craft brilliant work art leave shake head wonder |
| 12 | 0 | 3.4355 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | tomas alfredson let right original dark twist gory fantasy special hard classify merely exercise style brilliant piece amoral storytellings characters action defy logic common sense dont wanna spoil moment ll mean revenge moment story happens remind fantasy tale oskar kåre hedebrant year-old bully boy befriend develop innocent crush new neighbor eli klina leandersson happen vampired twist tale revenge pubescent love visual flair swim pool scene classic creative direct impressive performance young pair protagonists hollywood course didnt waste time announce upcoming remake lazy read subtitles likely remake turn pg-13 order money fill moral value prudish parent let kid watch dont vampired year ago ignore future bomb enjoy original youre treaty |
| 13 | 0 | 6.0132 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | people review idb focus sweet friendship year old human vampire leads huge element sweet story childhood friendship godfather story father concern worry kids true descriptions miss point think seen truth let right dark measure brilliant story tell week ago disturb echo stay refuse enjoy prior knowledge impossible review properly disclose plot elements warn spoilers ahead comment said character hokan key story end lead believe oskar step place fact reveal final scene dark nightmarish core blossom friendship witness prior offer redemption oskar path relationship eli takes håkan butcher boy old oskar food elik story storytelling high order dark whisper hint suggest questions allude play mind answered hokan pick victim presumably elias preference elias interest oskar elias true nature time briefly glimpse true physical age fact late middle age roughly age håkan gift oskar moments brutal slaughter child tormentors entranced glamour oskar happy happy happily sunset oskar presumably deaden soul destroy husk person hokan butcher fellow human provide eli food puzzle people think sweet fact conclusion utterly chilling oskar happily swap commonplace misery childhood bully fate truly hell earth happily smile shoot doesnt clue superlative heartily recommends tale succeed suggestion play imagination truly addition vampire cannon variation bram stokers brilliant original |
| 14 | 0 | 1.1521 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | fond vampire genre artistic poetic profound explore nature evil world child pure evil pure goodness somewhat discernible achieve astound array contrast allow evil coexist thought-provoking leave audience yearn read novel short gems need remake accord dont need wait 2010 watch |
| 15 | 0 | 0.6886 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | zombies zombies cheap outta original idea moves mandatory width train play beautiful girl sucker long-haired woman short skirts babe morality story ethic means played |
| 16 | 0 | 0.5323 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | different zombies army undead create industrial mishap result overwhelm chaos danger address morality root cause personally love pretty woman enjoy leg beautiful hair girl hat |
| 17 | 0 | 1.1526 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | train susan treat mind eyes look capitalism vs humanity man vs empathy remember time enjoy foreign act excellent effeminate young man story original definitely eye delicious cheerleader |
| 18 | 0 | 0.5609 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | dont youths fun action-packed dangerous elements innovative pretty girl importantly successful yup hollywood remake entry series redundant unnecessary inferior whitewash woman western taliban style ugly hollywood remakes paul fig jar jar abrams negotiate cut speak |
| 19 | 0 | 0.4347 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | zombie plenty thistle cheerleader splendid gods gift debut story peter end |
| 20 | 0 | 8.7146 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | ve guess director saw end inventive filmmaker work industry james wan brilliantly retro day deliver extremely stylistic visually strike stand tall classics virtually sex gore curse conjuring earn r-rating scare set 1971 conjuring focus marry paranormal researcher ed patrick wilson lorraine vera farmiga warren lecture college interest case theyre think retirement cue perron family parent roger iron livingstone carolyn lili taylor scare live live daughter claim evil rhode island home doesnt warrens discover perrons torment supernatural wanting short conjuring terrify ive seen trying erase torture porn crowd prove difficult director saw doubt finally look pg normally frown junkiest despite sex gore swearing james wants late slap r rate youre wonder frighten think mpa speak behalf day climax middle turn follow doesnt affect conjuring comic relief brilliantly place bring climax chance build fear startle wan understand psychology tension build suspense mere scene construction obviously note exorcist amityville inspiration derive real case file warren famous case date straying ironic style famous cabin woods surface story inventive assure work creative recent memory ive love james wan manage unoriginal haunted-house- possession story cases weave scares pacing sound design camera work describe precise james wan cinematographer john leonetti responsible look insidious fresh visual style unlike include wide shot movement cinematography absolutely stunning rely shaky cam realism leave unique feel separate visually narratively relate toone issue ive recent genre reveal demon insidious sinister example hook story fear unknown filmmaker audience wan definitely learn insidious conjuring apparition arendt reveal audience theyre far focus letting use imagination truly horrifying think filmmakers note mrs wan want complaint isnt wrong market screw trailer conjuring reveal scares avoid trailer command james wants twitter account hard miss tv spots wish fresh mind havenet trailer want stay marketing ultimately overall production value allow conjuring stand rot genre act impressive practical effect perfect classic feel wan time movies must-see summer rating 10let twitter thejoshl think conjuring |
| 21 | 0 | 4.0121 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | dont summon devil dont priest lucky conjuring preview screen brightest 2013 totally cold trailer story plot turn scary seen conjuring nail-biting hiding-behind-hands youve disappoint paranormal activity insidious likely deliver area failed tell supposedly true story paranormal investigators aim rid family property suspect supernatural visitations disprove turn creaky floorboard slam doors tackle head-on leap faith require buy theme youre okay play pretty genre confines particular incident bring deal describe surround spirit malevolent hide public fairly amityville-like circumstances family new house discover basement sealed boarded-up doorway surprise follows decide look basement surprise james wan manage tire theme haunt possession revive convincingly strange kind truly tension terror enjoy sinister recently equally scary lose bite end conjuring tempo fairly satisfy conclusion turn comical middle minutes fire cylinder head finale intentional lure false sense security work beautifully youre type poo-poo genre general canst suddenly convert believer enjoy classic exorcista amityville poltergeist guarantee conjuring wont disappoint |
| 22 | 0 | 4.4468 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | ism avid fan lately ive think isnt scare sinister skin appreciate james wants love saw insidious damn modern ghost story review state kinda lose momentum final act conjuring better scarier tense insidious ism gonna limb years classic rule making bad guy fully audience plus suspense tell throw water bottle screen sheer terror scare finally happened fake jump scares earn r rate blood sex profanity terror build scary masterpiece script acting direction style tone notch wants camera-work far movies choice set stroke genius feel authentic real story set present wouldnt effective scary style making costuming hair style throwback exorcist amityville extremely minor flaws length repetitiveness resolution happen fast dont care general flaws perfect havenet james wants conjuring ive time time scare badly dont miss amaze theater view experienced |
| 23 | 0 | 5.6439 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | key conjuring freshness evidence ream fan argue internet site new table fan hungry time conjuring right splinter deal admire day age stand tall proud ream remakes sequel teen friendly slasher haunt multiplex frequency days free gore sex automatically alienate portion lustful member fan based serve appetising scare cauldron suspense deliver plenty particular table forget based true story tag kind irrelevant new technological age sell gimmick mean story true play bit regardless hoax charge embellishments buy premises commit scary story director james wan reward compliant story essentially base investigation early seventy paranormal specialist ed lorraine warren aid perron family victim dreadful supernatural event rhode island home wan build deftly let perron family live believe dream home start happen wan build slow instances create palpable sense dread camera work intelligent moment maximum impact yearn warrens involved kill boo-jump scares place care marry superbly mount tension naturally clich convention haunt house strange smells creaky doors ominous cellar supplement wants talented knack scare effective production designs mysterious bruise literal leg pull breath hold game hide seek bona fide pant soil moments conjuring lesson sustain unease finale unleashed script devoid cheese pointless filler refresh sub-genre suffer problems joseph bishara musical score absolute nerve shredder refresh accompaniment doesnt resort telegraph shriek tell afraid overwhelm scene john leonetti cinematography gothic textures house outside lakeside farmhouses strong lead cast vera farmiga patrick wilson lili taylor ron livingston trump met critical box office success conjuring justify reputation superb haunt house |
| 24 | 0 | 5.0902 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | conjuring high class hard scare care character story compel feel interest time based true life events ed lorraine warren paranormal investigator set help family terrorize demon terrify case live hadnt share love place 70 agree completely director james wants point view impossible set present example teenage daughter haunt family picture demon diphones post post instagram basically demon turn victim human bad guys isnt perfect thought plot hole scare jump seat build tension let character start craziness set 70 feel 70 slow zoom style wouldnt expect modern believe james wan outdo ill far scary ive life thank james wan luck fast furious d recommend look guess feel audience feel watch exorcist dont look poltergeist time |
| 25 | 0 | 3.2186 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | pit human horrendous extra terrestrial end cheap imitation aliens series pitch black stand fine piece sci-fi excellent favorite aspect lighting beautifully employ different colors shade intensity light set mood lend unique feel different normal light generally subject vin diesel bring character life excellent manner skillfully avoid routine portrayal harden criminal riddick diesel character personal journey thankfully vin doesnt drop ball remainder cast exception talented gorgeous claudia black unknown turn marvelous performances animate diverse character unique quirk mannerisms pitch black perfect example resource excessive budget special effect adequate time mean sole focus high budget blockbusters use science fiction medium tell engaging provoke story tell mediocre story use flash science fiction |
| 26 | 0 | 4.0486 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | pitch black survival story survive hostile alien world hostile enemies task difficult near enemy survive group plot pitch black usual time different variations shine well-written characters group consist different people interest jacks boy secret fryd pilot hard time conscience johns bounty- hunter drug-habit imams holy man face fact god cruel riddick convict murderer learn value characters start live arendt paper genre start care characters reddick feel odd suppose bad guy line light pitch black think character familiar work casting rarely actor fit role radha mitchell suitable fryd cole hauser bring right cruelness sense responsibility johns impressive work vin diesel job riddick hand reddick creepy definitely dangerous deep character suppose survive pitch black friend close enemy closer |
| 27 | 0 | 1.8767 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | admit fan serenity reddick chronicles bite predetermine anticipation flip ism vin geisel fan somewhat mono-dimensional range characters embodiment riddick fit perfectly story play pace encumber gratuitous special effect fit cgi appropriate glare technical defect story start walk breathe air slight concern difference atmosphere microbe common flaw gloss stories canst harp detail radha mitchell smoking hot ridiculous frivolous tough season interplanetary merchant marine ships captain character fry nail perfectly remind andrea start hungarian assassin gilda tigers lady appear drop check heartbeat digress cases trouble grant star pitch black |
| 28 | 0 | 4.7084 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | doubt excite satisfy ive years plot print banal- ship crash desert planet suns survivor adjust landscape darkness fall monster appear pilot fryd moment cowardice descent atmosphere jettison passengers charge group enlist help convict murderer reddick lead darkness escape ship surgically enhance eye dark simple character complex three-dimensional conflict trait whoas whoas bathe performance uniformly superb radha mitchell fry steel leadership overcome fears prevent bloodshed cole hauser man reddick custody agendum idiosyncratic standards belong vin diesel reddick magnetic screen presence ive years face shadow doesnt deal draw attention draw attention speak moving diesel doesnt trivialise character easily heart gold reddick mean vicious man approach ship let glimpse tentative step care self care technically light effect excellent end spectrum overbright triple sunlight pitch darkness special effect riddick monsters point view add suspense sound effect monster fly ultrasound monster anatomically plausible suitably frightening editing tight jar time literally pad fades time re-orient open shoot end credit wit scene line dialogue single camera shoot important time understand ally caveat science solar system model impossible planets revolve suns vice versa doesnt affect human story havenet point |
| 29 | 0 | 1.5165 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain |
In [88]:
lsi_most_representative_docs = get_most_representative_docs(lsi_dom_topics_final, n_topics=20, lsi_flag=True)
lsi_most_representative_docs.style.applymap(color_green).applymap(make_bold)
Out[88]:
| Probability_Contribution | Topic_Keywords | Most_Representative_Document | |
|---|---|---|---|
| Topic_Number | |||
| 0 | 12.0119 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre | id eye years constantly check released trailer immediately hook need finally safely wait worth review point critic keyboard warrior complain baskin low plot hearing complaint grind gears devoid creativity generate interest plot basking designs dont fine low plot complaint absurd think masterful classic threadbare plot bat eyelash fact praise love basking writer director eternal craft wonderfully nightmarish understand fabric bad dream- dream wake canst baskin sense scene scene follow narrative plot thread detail stick thoroughly dreamlike quality dive deep dream realms character deliver spooky speech fate death camera fetishizes detail look irrelevant eye draw scour scene clue hint lurk shadows scene atmospheric moody uneasy vibe happening simple plot doesnt mean simple leave open imagination think concept marry usually mean killer monster isnt seen gory stuff happen screen true baskin let use imagination story theres obviously complex history antagonist think unnerve scary watch kill victims speak victims time protagonists team police officers happen stumble lair old abandon police station- clearly theyve small cult kill dozen dozen people extremely brutal ritualistic manners ive prop filmmakers create disturb look cult ive seen perfect cast cult leader meet cerrahoglu obviously wont mean audience mehmet screen credit god performance deeply unsettle extremely disturbing bright dark future movies new berryman love fondness genre actor humanly possible steal establish lead point meet dynamic chill screen presence evernol inspiration readily apparent genre fans clive barkery eli roth david lynch nicolas winding refn- manage blend eerie euro art-house vibe hardcore splatter flick vibe no- scratch doesnt blend them- riskier eerie euro art-house cinema smash rub face anxiety inducing gross- extreme gorer average splatter flick baskin bloody insane i- genre fan jaded hell doesnt mean canst recognize dement wild ride sick haunt concern mood instantly disturb imagery backstory plottings necessary story bit tell character interactions fine aspect baskin succeed overall excel leave memorable image average person wouldnt want let stick head baskin doesnt look cheap low budget immersive gritty readily apparent flaw chalk differ taste opinions people plot-heavy movies obviously let baskin bit doesnt mean thin plot flaw retrospect feel small locations restaurant highways old not-so-abandoned police station feel small happen location milk set surprisingly effective lesser hands ve absolute misfire slick piece gore-splatter cinema moody cinematography vibrant colors synth heavy score bring mind extra bloody satanic spin-dry wrong turn nicolas winding refund couldnt bad tried baskin destine overnight cult status genre immortality love |
| 1 | -47.9122 | sarah, juno, friend, cave, subconscious, action, deep, leave, stab, conscious, level, people, kill, event, dreamer, wreck, husband, story, run, death | whats interest deep mean symbol women sarah friend juno open sarah lose husband child horrific wreck course apparent accident cause husband distract upset affair juno sarah aware subconsciously juno betrayal lead death husband child place mentally emotionally acknowledge conscious level sarah awaken wreck sequence run darkness darkness symbolize action sarah reject run isnt ready classically chase dream dreamer pursue subconscious horrible form dreamer conscious effort stop run speak scary chase dreamers unconscious speak directly dreamer conscious dreamer drop ego defense running interact subconscious chasers gain new insight year pass juno friend spelunking lie cave discover want sarah cave complex juno sick mind sheds perform type penance doesnt sheds manipulate friend incredibly dangerous juno manipulator narcissistic point view people real juno feel going underground dream imagery subconscious basement bar fight club underwater sarah return darkness run earlier start experience hallucination waking dreams mainly daughters laugh subconscious pull event damage psyche wreck trap deep cave sarah deep deep subconscious readily apparent deep point cave deep level consciousness encounter cave people watch ll notice stop travel downward fall push downward choose cave people base ancient level sarah mind lack senses problem-solving tool-using skill strong aggressive primal force creatures key event happen juno sarah friend begin pick juno true character revealed stab friend accident leave help shock purpose whats interest soon juno reunite main group wont leave sarah missing act differently people character arendt watching juno person weak character key event sarah separate group end den cave dwellers heart low level subconscious juno original betrayal fully sarah conscious self friend juno stab bleed death tell sarah juno husband low level sarah understand juno juno type woman wreck marriage friend stab friend leave die lead friend certain death ego trip short juno destructive force sarah kill hurt friend end suffering action juno refused sarah empathize friend refuse leave pain escape juno juno feel feeling safety important friends suffering event happen room sarah kill cave dweller child observe mother grieve loss sarah commit sin juno mother kill sarah miss sarah life sarah need vengeance juno sarah eventually meet juno begin survivors fight cave dwellers sarah confront juno lies manipulations betrayals sarah stab juno leg leave cave dwellers action juno early stab friend leave different sarah conscious choice kill juno responsibility action juno avoid responsibility sequence sarah climb upward dark light symbolic return consciousness real world parallel hamlets miss sarah miss hamlets need action horribleness action recoil spend story point term |
| 2 | 35.0989 | people, battle, story, world, royale, school, youth, high, island, kill, idea, juno, sarah, videogame, hate, love, young, teenagers, die, problem | kenji fukasaku battle royale 2000 plenty past ive apart battle royale ism search battle royale affect people rabid fan battle royale people hate let tell battle royale exercise right explore idea idea poorly realized battle royale awesome hypothetically traumatic happen teenagers dont focus happen group high school student send abandon island kill bring bizarre idea fruition include civil unrest teenage anxiety nation literally terrorize youth set japan hit pretty close reality current problem japanese youth fact poorly receive government fear release incite riot act mayhem youth focus problem world young people volatile 20-30 year ago battle royale capture essence todays youth face circumstances friends kill friend bully survive time live videogame love play home note counter-strike half-life popular videogame mod example easily prepare young people reality weapons bullet clip mp5 assault rifle sound questions easily answer videogame today weapon street world hand child young years-old videogame set create comfortable experience weapons videogame necessarily people want gun familiarity guns mention love play counter-strike continue play future dont hate game ism point present fairly realistic portrayal weapons problem survivor island massacre add extra pressure unprepared child fight lives truly shock actor actress select portray teen age characters arendt age 20-30 somethings happen look young literally teenagers flick bite comment live century time frequently fear country outside forces terrorism forefront average north americanos mind world trade center attack cents endless coverage event easily problem international events headline grabber focus young people fill `rage unleash anger helpless peer array weapon mainly guns school shooting shock world child start kill peers battle royale mean trivialize school shooting youth violence examination length government order discipline youth ludicrous idea character stay true form profess hold crush dye breath naively trust theyve admire popular kids sick strange beautiful familiar different completely engaging people feel plot silly dialogue hammy nonsense time naysayer praise braveheart honest portrayal scotland historical hero love braveheart think bogus certain battle event happen william wallace man look rape kill woman small child kind feel sell wallace hero battle royalet draw fictitious character plot far interest think life kill friend high school stick island death day refuse answer question sicken think feel disturb kid battle royalet whats wrong pick lottery end island japan exist battle royalet year random high school class pick events lead believe youth japan bad seed doesnt apply class follows intent purposes innocent dialogue character poignant real totally innocent literally limit vocabulary understand world furthermore mention earlier character profess love classmate love high school weird time anybody awkward time experience misunderstandings people high school learn truth people think high school theres social wall stop real open communication people island force unchecked emotion feeling flow character death horizon label dialogue lousy circumstances obviously intelligent well-organized people world exist high school teenager brash foolish irresponsibly reckless theyve learn experienced rarely experienced teenagers island kill learn new wont matt consider theyll soon dead naturally insane mutter math equation teacher promise valuable real world feel need fulfill sexual desires want die virgin right situation spend hour alive civilize possible purpose game affect teenagers hurry battle isnt finish day die easy people charge low-jacked teenager collar explode head clean default explosion open jugular bleed die hope dream future grisly doesnt specifically cater gore hounds disturb death scene moment idea shock theres need excessive fact upset want bloody parade carnage reason form entertainment exist idea sicken compel satisfy layer visual end street sceptic people unable maturely grasp concept people hate canst blame hollywood dominant authority filmmaking world greatly expressive thought-provoking medium trite bore things recycle repackaged re-sold re-distributed point people hardly accept new radical different safe generics commercial reason existence appear highly questionable battle royale isnt change world wish damage place challenge socio-political norm subtitles thats alright matt dont press attention leonardo dicaprio enjoy battle royale kenji fukasaku direct adapt screen son kenta able rest peace man die january 12th 2003 years-old want died wish fan don t hate matt hard theres better j symister 2002 |
| 3 | -23.0985 | water, ring, dark, original, yoshimi, watch, scene, work, feel, world, girl, ikuko, think, time, beast, child, nakada, young, form, people | yoshimi matsubara hitomi kuroki middle nasty divorce husband unio hamada fumiyo kohinata issue contention daughter ikuko ario kanno unio accuse yoshimi unstable point yoshimi award temporary custody ikuko apartment ikuko live pick less-than-ideal apartments affordable soon strange occurrence begin yoshimi bedroom ceil develope water stain mysterious puddle water appear different locations unusual item appearing despite attempt discard yoshimi periodically strange girl glimpses ikuko begin act oddly yoshimi work sheds trouble balance care ikuko things spiral control problem yoshimi divorce sinister supernatural despite dark waters relatively overt similarity numb filmic works director hide nakata successful films--at fame ring 1998 better awfully close dark water easily raise score subsequent viewings facet open example fact check shortly write review re-watched beginning open credit extremely eerie impact doesnt hit youve fully realize youre look watch shot similarity include thematic resemblance ring shouldnt surprise consider nakada director base novel man japanese stephen king japanese press--koji suzuki ring dark waters menace form young haired japanese girl frequent mysterious appearances girls focus irony--they suppose cute turn performance adult fellow cast innocent girl menace unnerving menace accompany water water important symbolism ring took venture guess nakada andor suzuki fear water impersonal took water powerful force easily adapt surrounding easily mold permeate world visual symbol kamis shinto essence beingness permeate things godlike soul dead human tsumi pollution form kami symbolically cleanse water important symbolic commonality share ring dark water claustrophobic spaces occur ring form closet crawl spaces dark water elevator structure ll realize importance near end water combine elevator enable nakada nice nod stanley kubrick shining 1980 scene similarity ring dark water concern familial problem concern fact symbolic metaphor familial problem ringu ring clear nakata latest american ring film--the ring 2005 feature young mother struggle maintain normal existence child dark water easy element mere metaphor yoshimi psychological decline effect daughter echo problematic childhood--we learn parent divorce young open dramatic scene yoshimi child wait school pick hear comment mother bad dark water focus nakata deliberate pace perfect spooky event subtle unnerving nakada achieve amaze build-ups scene elevator near end frighten reveal reveal work nakada build tension stretch pregnant pause viewer ready burst scene dark water succeed story relatively simple straightforward unlike typical american story told implication viewer frequently leave figure decision event base seemingly innocuous comment antecedent scene follow relationship scenario change follow scene words assumption happened sound complex aim wonderfully achieved simplify event screen famous asian dream logic present supernatural events doesnt usurp plot continue gradually hone build tension yoshimi husband ikuko mystery girl apartment complex ending comment element profound theyve changed uncanny poignant |
| 4 | -17.745 | story, original, room, work, beast, vampire, ghost, 1408, battle, watch, french, school, character, scene, remake, le, creature, frontal, mike, genre | 1765 stalk mountain south-western france beast pounce human animal terrible ferocity beast notorious king france dispatch envoy happen kill creature end beast gevaudan kill 100 people day entirely wolf hyena supernatural shepherd life-expectancy red-suited guy star trek beast popular myth france albeit root firmly reality somewhat surprisingly outside world incredibly categorising le pace des loups tricky ill period costume martial-arts werewolf surprisingly piece work provide dont concentrate hard previously period costume martial-arts werewolf mystery expect plenty imitation future taking beast start point quickly diverge historical fact step pace introduce heroes gregoire frontal samuel le bihan mani mark dacascos midst torrential storm culminate magnificently stage fights frontal dispatch king beast frontal represent new rational world enlightenment force confront backward superstitious france outside capital mania iroquois shaman hunter befriend frontal whilst adventure americas bring type wisdom entirely time america dark mysterious place home fear europeans course shortly home republicanism sweep france remake old world new image le pace des loups wear republican colour sleeve use conflict rationalism stereotypical backward villager drive home point old-fashioned territory source plot guvaudan sort village inhabitant sleepy hollow creeps england christopher lee lord manor peter cushing priest le pace des loups strong french language cast possible mix veteran talent dominate priest sardis jean-fran stévenin saturnine jean francois vincent cassel cripple hunter explorer rapidly dangerous beast scornful change paris seek shield world future remainder population stupid indolent superstitious evil hold new rational world cities beast extension physical monster beast projection villager hatred bigotry strong female role unusual movies different characters strikingly elegant hypnotic courtesan sylvia monica bellucci play role seductress frigid professionalism world woman wit protect dangerous far appears unsure help hinder heroes mysterious captivating complete contrast innocent fragile astonishingly beautiful madeline emilie dequenne young sister protective jean francoise surrounded evil prejudice superstition clearly romantic heroine intend represent french republic symbol frontal fall hopelessly love witty charm woman risk conflict jean francois lead fantastic share chemistry reminiscent relationship butch sundance le frontal wise needed sensational think mani human mani man word utterly dominate screen present needless fantastic fighters live die creature fortunately delivers wisely chance look animal nasty viewers mind fill details creature surprisingly sparingly viewer expect pounce doesn t minute appear wonderful shock stuff reminiscent alien whilst design animal creature workshop perfect cgi work standard expect couple daylight shot well-below nighttime work outstanding shoot creature stalk fog hero effective cgi work cinematically work bear resemblances ridley scotts gladiator luscious slow character-forming scene mix frantic camera work action scenes strange hybrid style work exceptionally frantic camera digital trick excellent effect flashback scene striking strongly solarised effect otherworldly texture designer previously involve merchant ivory production luxurious interior scene bite detail period piece special word costume use sumptuous fabric possible deal light candle fire light fill screen warm orange flesh tone miss chance flesh contrast exterior shot frequently chill blue wash hues french countryside look hostile world conceal form dark secrets countryside magnificently different stereotypical french landscapes tragically splendour play minuscule audiences people whilst queue american pie stretch auditoriums favour foreign language people think french cinema involve middle age peasant smoke gauloises whilst argue fine point philosophy revelation 140 minute run plot twist near end difficult accept wince accept doesnt outstay welcome english-speaking market subtitled sadly workmanlike translation witty dialogue isnt translated shame script poor french speaker sparkles numb misspelling grammatical error subtitle catch earlier read subtitles isnt art doesnt redefine genre doesnt preach nature irrational learn apart dont split group basement check light deliver hour solid entertainment ll silly grin face want finally word praise imaginative dissolve shot womans breast fade mountains doubt woman world eager christophe gans eiffel towering short le pace des loups paw |
In [89]:
lsi_topic_dist = get_topic_distribution(lsi_dom_topics_final, lsi_most_representative_docs, n_topics=20)
lsi_topic_dist.style.applymap(color_green).applymap(make_bold)
Out[89]:
| Documents_per_Topic | Percent_of_Total_Corpus | Topic_Keywords | |
|---|---|---|---|
| Topic_Number | |||
| 0 | 988 | 0.988 | story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre |
| 1 | 4 | 0.004 | sarah, juno, friend, cave, subconscious, action, deep, leave, stab, conscious, level, people, kill, event, dreamer, wreck, husband, story, run, death |
| 2 | 4 | 0.004 | people, battle, story, world, royale, school, youth, high, island, kill, idea, juno, sarah, videogame, hate, love, young, teenagers, die, problem |
| 3 | 2 | 0.002 | water, ring, dark, original, yoshimi, watch, scene, work, feel, world, girl, ikuko, think, time, beast, child, nakada, young, form, people |
| 4 | 2 | 0.002 | story, original, room, work, beast, vampire, ghost, 1408, battle, watch, french, school, character, scene, remake, le, creature, frontal, mike, genre |
In [90]:
#token pattern prevents default splitting of hyphenated words
vectorizer = CountVectorizer(token_pattern='(?u)\\b[\\w-]+\\b', analyzer='word', max_features=5000)
x_counts = vectorizer.fit_transform(all_reviews_joined)
transformer = TfidfTransformer(smooth_idf=False)
x_tfidf = transformer.fit_transform(x_counts)
In [91]:
lsi_svd_2D = TruncatedSVD(n_components=2, algorithm='randomized', n_iter=500, random_state=222)
lsi_2D = lsi_svd_2D.fit_transform(x_tfidf.T)
In [92]:
svd_2D_scatter(lsi_2D, color_scale_dimension=1)
In [93]:
lsi_svd_3D = TruncatedSVD(n_components=3, algorithm='randomized', n_iter=500, random_state=222)
lsi_3D = lsi_svd_3D.fit_transform(x_tfidf.T)
In [94]:
svd_3D_scatter(lsi_3D, color_scale_dimension=0)
NMF (Non-Negative Matrix Factorization)¶
Cf. https://papers.nips.cc/paper/1861-algorithms-for-non-negative-matrix-factorization.pdf¶
In [95]:
#token pattern prevents default splitting of hyphenated words
vectorizer = CountVectorizer(token_pattern='(?u)\\b[\\w-]+\\b', analyzer='word', max_features=5000)
x_counts = vectorizer.fit_transform(all_reviews_joined)
transformer = TfidfTransformer(smooth_idf=False)
x_tfidf = transformer.fit_transform(x_counts)
xtfidf_norm = normalize(x_tfidf, norm='l1', axis=1)
In [96]:
mod_list_nmf_umass, coh_list_nmf_umass = compute_coherence_values(NMF, dictionary, all_reviews_text, max_topics=20,
transformed_vectorizer=vectorizer, tfidf_norm=xtfidf_norm,
min_topics=2, stride=3, n_top_words=20, measure='u_mass',
nmf_flag=True)
mod_list_nmf_cv, coh_list_nmf_cv = compute_coherence_values(NMF, dictionary, all_reviews_text, max_topics=20,
transformed_vectorizer=vectorizer, tfidf_norm=xtfidf_norm,
min_topics=2, stride=3, n_top_words=20, measure='c_v',
nmf_flag=True)
Coherence of model with 2 topics has been computed Coherence of model with 5 topics has been computed Coherence of model with 8 topics has been computed Coherence of model with 11 topics has been computed Coherence of model with 14 topics has been computed Coherence of model with 17 topics has been computed All coherence values have been computed for the <class 'sklearn.decomposition.nmf.NMF'> using the U_MASS measure Coherence of model with 2 topics has been computed Coherence of model with 5 topics has been computed Coherence of model with 8 topics has been computed Coherence of model with 11 topics has been computed Coherence of model with 14 topics has been computed Coherence of model with 17 topics has been computed All coherence values have been computed for the <class 'sklearn.decomposition.nmf.NMF'> using the C_V measure
In [97]:
best_num_nmf_topics = show_best_num_topics('NMF', coh_list_nmf_umass, coh_list_nmf_cv, max_topics=20, min_topics=2, stride=3)
print('The most coherent number of NMF topics to use is: {}'.format(best_num_nmf_topics))
The most coherent number of NMF topics using the U_MASS measure is: 17 The most coherent number of NMF topics using the C_V measure is: 2 The most coherent number of NMF topics to use is: 2
In [98]:
nmf = NMF(n_components=best_num_nmf_topics, init='nndsvd', random_state=222)
nmf.fit(xtfidf_norm)
Out[98]:
NMF(alpha=0.0, beta_loss='frobenius', init='nndsvd', l1_ratio=0.0, max_iter=200, n_components=2, random_state=222, shuffle=False, solver='cd', tol=0.0001, verbose=0)
In [99]:
nmf_df = get_nmf_topics(nmf, best_num_nmf_topics, 20)
nmf_df
Out[99]:
| Topic # 01 | Topic # 02 | |
|---|---|---|
| 0 | watch | summary |
| 1 | story | describe |
| 2 | dont | funny |
| 3 | time | scary |
| 4 | feel | plot |
| 5 | people | story |
| 6 | think | phobia |
| 7 | end | salvage |
| 8 | character | backpackers |
| 9 | original | ward |
| 10 | scare | gory |
| 11 | ive | bias |
| 12 | ism | end |
| 13 | look | insidious |
| 14 | scene | gods |
| 15 | fan | gift |
| 16 | love | plenty |
| 17 | leave | peter |
| 18 | scary | novice |
| 19 | act | purikitpanya |
In [100]:
nmf_hellinger_topic_distances = get_most_similar_topics(nmf, nmf_flag=True, columns=nmf_df.columns)
pretty_print(nmf_hellinger_topic_distances)
Hellinger Distances between topics are: ______________________________________________________________________ (0.8287134542069157, 'Topic # 01', 'Topic # 02') ______________________________________________________________________
NMF: Second Pass with U_Mass Coherence optimized number of components¶
In [101]:
nmf_2 = NMF(n_components=17, init='nndsvd', random_state=222)
nmf_2.fit(xtfidf_norm)
Out[101]:
NMF(alpha=0.0, beta_loss='frobenius', init='nndsvd', l1_ratio=0.0, max_iter=200, n_components=17, random_state=222, shuffle=False, solver='cd', tol=0.0001, verbose=0)
In [102]:
nmf_df_2 = get_nmf_topics(nmf_2, 17, 20)
nmf_df_2
Out[102]:
| Topic # 01 | Topic # 02 | Topic # 03 | Topic # 04 | Topic # 05 | Topic # 06 | Topic # 07 | Topic # 08 | Topic # 09 | Topic # 10 | Topic # 11 | Topic # 12 | Topic # 13 | Topic # 14 | Topic # 15 | Topic # 16 | Topic # 17 | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | feel | summary | gods | zombies | vs | insidious | ghost | phobia | ism | pirate | james | ring | hill | evil | conjuring | wrong | fun |
| 1 | character | describe | gift | morality | man | whats | sci-fi | segment | think | poses | watch | ju-on | silent | remake | wow | downhill | jar |
| 2 | story | funny | peter | mandatory | empathy | scary | background | funny | ive | waste | split | scary | game | pure | scare | answered | entry |
| 3 | time | scary | plenty | long-haired | foreign | aftermath | people | kinda | review | significance | dont | owe | look | need | jump | don | hollywood |
| 4 | scene | plot | debut | played | humanity | magically | complicate | backpackers | eden | disease | personalities | exorcista | think | yearn | cinema | boring | sundance |
| 5 | end | story | zombie | beautiful | train | grab | depend | salvage | thrillers | trust | mcavoy | agree | mystery | vampire | sequel | rating | park |
| 6 | real | recommend | end | train | eyes | intense | creativity | ward | watch | wood | different | reviews | isnt | poetic | conjure | t | seriously |
| 7 | people | life | story | woman | remember | scared | special | suspense | worth | bunch | saw | powerful | drama | thought-provoking | ive | m | style |
| 8 | genre | korean | infect | means | treat | risk | magnificent | gory | read | truth | award | water | doom | profound | incredible | list | redundant |
| 9 | director | interesting | suspenseful | girl | eye | door | relate | bias | ill | answer | wait | uncomfortable | visuals | gems | boredom | hole | inferior |
| 10 | new | love | virus | cheap | mind | recently | asian | stories | actress | question | win | unlike | resident | achieve | youve | predictable | day |
| 11 | twist | strongly | corpses | short | excellent | jump | effect | story | better | wouldnt | comment | supernatural | said | contrast | anymore | couldnt | abrams |
| 12 | act | disturb | zombies | idea | definitely | entirely | expectation | close | bad | kill | played | stuff | crowd | watch | lucky | night | innovative |
| 13 | didnt | dont | chainsaw | original | original | hear | comment | messy | fan | slow | mistake | cheap | video | artistic | teenager | large | importantly |
| 14 | doesnt | works | texas | play | enjoy | genuine | hollywood | novice | slow | night | today | shot | welcome | explore | lets | expect | unnecessary |
| 15 | house | troll | slight | address | young | talk | dont | end | thriller | happen | wont | decide | crap | dead | disappoint | question | remakes |
| 16 | family | polish | massacre | army | act | fail | enjoy | humorous | ending | scary | performance | experience | visual | hindi | honest | minute | paul |
| 17 | work | ideal | flesh | hat | look | scenes | typical | cool | build | dont | shyamalan | dark | pop | nature | usually | leave | ugly |
| 18 | want | attractive | quietly | overwhelm | time | remind | terrify | scary | die | people | convince | watch | intelligent | novel | scary | isnt | pretty |
| 19 | tension | watched | random | leg | story | room | believe | marsha | hard | time | write | ism | memorable | somewhat | couldnt | wait | red |
In [103]:
nmf_hellinger_topic_distances_2 = get_most_similar_topics(nmf_2, nmf_flag=True, columns=nmf_df_2.columns)
pretty_print(nmf_hellinger_topic_distances_2)
Hellinger Distances between topics are: ______________________________________________________________________ (0.9584644507879246, 'Topic # 03', 'Topic # 06') (0.9424645212814688, 'Topic # 03', 'Topic # 15') (0.9399235670404483, 'Topic # 02', 'Topic # 05') (0.9346295431141141, 'Topic # 03', 'Topic # 05') (0.9344291361702227, 'Topic # 03', 'Topic # 13') (0.9322648514116035, 'Topic # 04', 'Topic # 06') (0.9315124306651026, 'Topic # 02', 'Topic # 04') (0.9301052202804005, 'Topic # 03', 'Topic # 16') (0.9294710886755139, 'Topic # 08', 'Topic # 10') (0.9291101049170272, 'Topic # 03', 'Topic # 07') (0.9291003746279596, 'Topic # 05', 'Topic # 10') (0.9289175001415316, 'Topic # 04', 'Topic # 16') (0.9282824812843677, 'Topic # 04', 'Topic # 13') (0.9281640990078213, 'Topic # 04', 'Topic # 10') (0.9258252741428218, 'Topic # 06', 'Topic # 17') (0.9251840639484765, 'Topic # 03', 'Topic # 14') (0.924785695848468, 'Topic # 06', 'Topic # 08') (0.9209535866682126, 'Topic # 03', 'Topic # 08') (0.9199740730254865, 'Topic # 08', 'Topic # 16') (0.9196165653244697, 'Topic # 03', 'Topic # 10') (0.9193324527539374, 'Topic # 02', 'Topic # 03') (0.9192197190729483, 'Topic # 02', 'Topic # 10') (0.917734398614864, 'Topic # 02', 'Topic # 14') (0.9173162625447314, 'Topic # 03', 'Topic # 17') (0.9168218643907617, 'Topic # 02', 'Topic # 16') (0.9161233622399915, 'Topic # 04', 'Topic # 15') (0.9133152848397521, 'Topic # 03', 'Topic # 09') (0.9116331138366993, 'Topic # 02', 'Topic # 06') (0.911593557277736, 'Topic # 08', 'Topic # 13') (0.9114488871983222, 'Topic # 03', 'Topic # 11') (0.9113735217700517, 'Topic # 08', 'Topic # 12') (0.908458108265172, 'Topic # 02', 'Topic # 15') (0.9064553140855598, 'Topic # 03', 'Topic # 12') (0.9062050177358214, 'Topic # 04', 'Topic # 08') (0.9059031502277907, 'Topic # 10', 'Topic # 17') (0.9058733302866112, 'Topic # 04', 'Topic # 12') (0.9056778335861626, 'Topic # 05', 'Topic # 06') (0.9055958689354423, 'Topic # 08', 'Topic # 14') (0.9044965853974102, 'Topic # 02', 'Topic # 07') (0.9043340941387561, 'Topic # 04', 'Topic # 09') (0.903994283672788, 'Topic # 03', 'Topic # 04') (0.9038715444306065, 'Topic # 05', 'Topic # 16') (0.9037995829889024, 'Topic # 13', 'Topic # 17') (0.903413811906169, 'Topic # 04', 'Topic # 05') (0.9020537456963644, 'Topic # 02', 'Topic # 11') (0.901835879590631, 'Topic # 04', 'Topic # 07') (0.9012207668445605, 'Topic # 05', 'Topic # 08') (0.8999494677207568, 'Topic # 04', 'Topic # 11') (0.8990402272302405, 'Topic # 10', 'Topic # 11') (0.8980293628586249, 'Topic # 15', 'Topic # 17') (0.897705789738865, 'Topic # 02', 'Topic # 12') (0.897303524108698, 'Topic # 05', 'Topic # 11') (0.8971279939937775, 'Topic # 06', 'Topic # 11') (0.8969129114284167, 'Topic # 04', 'Topic # 17') (0.8967706373411575, 'Topic # 08', 'Topic # 11') (0.896114020736355, 'Topic # 05', 'Topic # 12') (0.8944365471996264, 'Topic # 02', 'Topic # 13') (0.8939917871520082, 'Topic # 07', 'Topic # 08') (0.892794574546296, 'Topic # 02', 'Topic # 17') (0.89276442270781, 'Topic # 02', 'Topic # 09') (0.8925193082712505, 'Topic # 12', 'Topic # 17') (0.8921381443325341, 'Topic # 08', 'Topic # 17') (0.8901026918028777, 'Topic # 06', 'Topic # 14') (0.8889367919218137, 'Topic # 05', 'Topic # 13') (0.88844179547972, 'Topic # 16', 'Topic # 17') (0.888017458625833, 'Topic # 06', 'Topic # 13') (0.8877450620813365, 'Topic # 10', 'Topic # 14') (0.8865843692498548, 'Topic # 07', 'Topic # 16') (0.8865535168589743, 'Topic # 05', 'Topic # 14') (0.8859040418084831, 'Topic # 10', 'Topic # 15') (0.8854040099500482, 'Topic # 13', 'Topic # 16') (0.8849448321471554, 'Topic # 06', 'Topic # 16') (0.8846477792497829, 'Topic # 04', 'Topic # 14') (0.8839170558595878, 'Topic # 08', 'Topic # 15') (0.8826106830401623, 'Topic # 13', 'Topic # 14') (0.8825175820868869, 'Topic # 05', 'Topic # 17') (0.8822830273699803, 'Topic # 05', 'Topic # 15') (0.8817729623835481, 'Topic # 08', 'Topic # 09') (0.8815908166368477, 'Topic # 07', 'Topic # 11') (0.8814407765137757, 'Topic # 07', 'Topic # 14') (0.8810655361398263, 'Topic # 11', 'Topic # 16') (0.881007864900373, 'Topic # 14', 'Topic # 16') (0.8805133571218197, 'Topic # 11', 'Topic # 13') (0.8794852605253304, 'Topic # 06', 'Topic # 09') (0.8769534447500179, 'Topic # 11', 'Topic # 17') (0.8769305197381267, 'Topic # 05', 'Topic # 07') (0.8764848171790806, 'Topic # 07', 'Topic # 17') (0.8761736952013941, 'Topic # 07', 'Topic # 10') (0.8741326480827946, 'Topic # 06', 'Topic # 07') (0.8740462766943192, 'Topic # 12', 'Topic # 14') (0.8737575271325964, 'Topic # 10', 'Topic # 12') (0.8737189878294761, 'Topic # 14', 'Topic # 15') (0.873350406073119, 'Topic # 15', 'Topic # 16') (0.8719561260038503, 'Topic # 07', 'Topic # 13') (0.8698688361693404, 'Topic # 12', 'Topic # 13') (0.8689094667820856, 'Topic # 07', 'Topic # 12') (0.8685083498274481, 'Topic # 11', 'Topic # 12') (0.8682413384257885, 'Topic # 12', 'Topic # 16') (0.867391360926769, 'Topic # 10', 'Topic # 13') (0.8653900439144379, 'Topic # 09', 'Topic # 10') (0.8624647540171855, 'Topic # 06', 'Topic # 10') (0.8623928015184431, 'Topic # 11', 'Topic # 15') (0.8609140296129003, 'Topic # 14', 'Topic # 17') (0.8605266168357723, 'Topic # 09', 'Topic # 13') (0.8598861235951925, 'Topic # 09', 'Topic # 17') (0.8597004573353824, 'Topic # 07', 'Topic # 09') (0.8578453645250446, 'Topic # 09', 'Topic # 14') (0.857427955680357, 'Topic # 06', 'Topic # 12') (0.8571208200575151, 'Topic # 13', 'Topic # 15') (0.8561277260411315, 'Topic # 10', 'Topic # 16') (0.8554387815110195, 'Topic # 07', 'Topic # 15') (0.8550259091228346, 'Topic # 11', 'Topic # 14') (0.8531370139921345, 'Topic # 02', 'Topic # 08') (0.851784942370877, 'Topic # 01', 'Topic # 03') (0.8499604120234221, 'Topic # 06', 'Topic # 15') (0.8499454529807071, 'Topic # 01', 'Topic # 02') (0.8484159121000783, 'Topic # 09', 'Topic # 16') (0.8483299547608233, 'Topic # 09', 'Topic # 15') (0.8444322887790761, 'Topic # 05', 'Topic # 09') (0.8442968743292932, 'Topic # 09', 'Topic # 11') (0.8384120417252762, 'Topic # 09', 'Topic # 12') (0.8328400557963762, 'Topic # 12', 'Topic # 15') (0.8296730150866366, 'Topic # 01', 'Topic # 04') (0.8267393167070708, 'Topic # 01', 'Topic # 08') (0.8178112828683857, 'Topic # 01', 'Topic # 06') (0.8174981747238305, 'Topic # 01', 'Topic # 10') (0.7956580202009549, 'Topic # 01', 'Topic # 05') (0.7947136346996182, 'Topic # 01', 'Topic # 16') (0.7867317183203042, 'Topic # 01', 'Topic # 12') (0.7826714712806262, 'Topic # 01', 'Topic # 07') (0.7747618350774251, 'Topic # 01', 'Topic # 17') (0.7717195114488037, 'Topic # 01', 'Topic # 13') (0.7695156200854026, 'Topic # 01', 'Topic # 15') (0.7671204907764337, 'Topic # 01', 'Topic # 11') (0.7526477622365153, 'Topic # 01', 'Topic # 09') (0.7507159579845191, 'Topic # 01', 'Topic # 14') ______________________________________________________________________
In [104]:
svd = TruncatedSVD(n_components=2)
documents_2d = svd.fit_transform(x_tfidf)
df = pd.DataFrame(columns=['x', 'y', 'document'])
df['x'], df['y'], df['document'] = documents_2d[:,0], documents_2d[:,1], range(len(all_reviews_text))
source = ColumnDataSource(ColumnDataSource.from_df(df))
#alternative syntax
#source = ColumnDataSource({'x': np.array([documents_2d[:,0]]), 'y': np.array([documents_2d[:,1]]), 'document': np.array([range(len(all_reviews_text))])})
labels = LabelSet(x="x", y="y", text="document", y_offset=8,
text_font_size="8pt", text_color="#555555",
source=source, text_align='center')
plot = figure(plot_width=1200, plot_height=1000)
plot.circle("x", "y", size=12, source=source, line_color="black", fill_alpha=0.8)
plot.add_layout(labels)
#Cf. JSON Serialization issue:
# https://github.com/bokeh/bokeh/issues/5439
# https://github.com/bokeh/bokeh/issues/6222
# https://github.com/bokeh/bokeh/issues/7523
try:
show(plot, notebook_handle=True)
except Exception as e:
print('Note!: {}'.format(e.__doc__))
print(e)
In [105]:
svd = TruncatedSVD(n_components=2)
words_2d = svd.fit_transform(x_tfidf.T)
df = pd.DataFrame(columns=['x', 'y', 'word'])
df['x'], df['y'], df['word'] = words_2d[:,0], words_2d[:,1], vectorizer.get_feature_names()
source = ColumnDataSource(ColumnDataSource.from_df(df))
labels = LabelSet(x="x", y="y", text="word", y_offset=8,
text_font_size="8pt", text_color="#555555",
source=source, text_align='center')
plot = figure(plot_width=1200, plot_height=1000)
plot.circle("x", "y", size=8, source=source, line_color="black", fill_alpha=0.8)
plot.add_layout(labels)
#Cf. JSON Serialization issue:
# https://github.com/bokeh/bokeh/issues/5439
# https://github.com/bokeh/bokeh/issues/6222
# https://github.com/bokeh/bokeh/issues/7523
try:
show(plot, notebook_handle=True)
except Exception as e:
print('Note!: {}'.format(e.__doc__))
print(e)
LDA Visualization: T-distributed Stochastic Neighbor Embedding with PCA initialization¶
Cf. http://www.jmlr.org/papers/volume9/vandermaaten08a/vandermaaten08a.pdf¶
In [106]:
lda_base_model = lda.LDA(n_topics=10, n_iter=1000, random_state=111)
In [109]:
doc_x_tops = lda_base_model.fit_transform(x_counts)
In [110]:
#doc_x_tops.shape
In [111]:
#setting threshold above which documents allocated to topics are 'confidently' displayed
#threshold = 0.5
#idx of doc above threshold
#_idx = np.amax(doc_x_tops, axis=1) > threshold
#doc_x_tops = doc_x_tops[_idx]
In [112]:
# angle close to 1 prioritizes speed over accuracy
tsne_model = TSNE(n_components=2, verbose=1, random_state=222, angle=.1, init='pca')
# 10D -> 2D
tsne_lda = tsne_model.fit_transform(doc_x_tops)
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 1000 samples in 0.001s... [t-SNE] Computed neighbors for 1000 samples in 0.037s... [t-SNE] Computed conditional probabilities for sample 1000 / 1000 [t-SNE] Mean sigma: 0.150693 [t-SNE] KL divergence after 250 iterations with early exaggeration: 69.865005 [t-SNE] Error after 1000 iterations: 0.866557
In [113]:
#20 colors
colormap = np.array([
"#1f77b4", "#aec7e8", "#ff7f0e", "#ffbb78", "#2ca02c",
"#98df8a", "#d62728", "#ff9896", "#9467bd", "#c5b0d5",
"#8c564b", "#c49c94", "#e377c2", "#f7b6d2", "#7f7f7f",
"#c7c7c7", "#bcbd22", "#dbdb8d", "#17becf", "#9edae5"
])
#number of displayed keywords
n_top_words = 10
In [114]:
#Cf. JSON Serialization issue:
# https://github.com/bokeh/bokeh/issues/5439
# https://github.com/bokeh/bokeh/issues/6222
# https://github.com/bokeh/bokeh/issues/7523
plot_tsne(all_reviews_joined, lda_base_model, vectorizer, doc_x_tops, tsne_lda, colormap, n_top_words=10)
In [ ]: